diff --git a/.github/workflows/cancel-closed-pr-runs.yml b/.github/workflows/cancel-closed-pr-runs.yml new file mode 100644 index 0000000000..0e07039ac9 --- /dev/null +++ b/.github/workflows/cancel-closed-pr-runs.yml @@ -0,0 +1,37 @@ +name: "Cancel runs of closed PRs" + +# Merging or closing a PR leaves its pull_request runs going: concurrency +# groups cancel a run only when a newer push arrives. pull_request_target gives +# this job a token that can cancel runs, fork PRs included. It checks out and +# runs no PR code. +on: + pull_request_target: + types: [closed] + +permissions: + actions: write + pull-requests: read + +jobs: + cancel: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Cancel the PR's queued and running pull_request runs + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR: ${{ github.event.pull_request.number }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + run: | + set -euo pipefail + # Match the PR's own commits: forks reuse branch names such as master. + shas=$(gh api --paginate "repos/$GH_REPO/pulls/$PR/commits" --jq '.[].sha') + for status in queued in_progress waiting pending requested; do + gh run list --branch "$HEAD_REF" --event pull_request --status "$status" \ + --limit 200 --json databaseId,headSha --jq '.[] | "\(.databaseId) \(.headSha)"' + done | while read -r id sha; do + grep -qx "$sha" <<<"$shas" || continue + echo "cancel run $id at $sha" + gh run cancel "$id" || true + done diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index 45619a21e0..4336a3ce49 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -8,6 +8,7 @@ on: - 'seaweed-volume/**' - 'seaweed-common/**' - 'seaweed-worker/**' + - 'seaweed-vector/**' - 'docker/**' - 'go.mod' - 'go.sum' @@ -50,7 +51,8 @@ jobs: ~/.cargo/git seaweed-volume/target seaweed-worker/target/${{ matrix.target }}/release - key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} + seaweed-vector/target/${{ matrix.target }}/release + key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }} restore-keys: | rust-docker-dev-${{ matrix.target }}- @@ -83,6 +85,12 @@ jobs: cargo build --release -p weed-lance-worker --target ${{ matrix.target }} cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Build the vector engine + run: | + cd seaweed-vector + cargo build --release -p weed-vector-server --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: @@ -90,6 +98,7 @@ jobs: path: | weed-volume-normal-${{ matrix.arch }} weed-worker-${{ matrix.arch }} + weed-vector-${{ matrix.arch }} build-dev-containers: needs: [build-rust-binaries] @@ -127,8 +136,17 @@ jobs: echo "Placed pre-built Rust worker for ${arch}" fi done + mkdir -p docker/weed-vector-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-vector-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}" + echo "Placed pre-built vector engine for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ ls -la docker/weed-worker-prebuilt/ + ls -la docker/weed-vector-prebuilt/ - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 668c6046f6..f5a445e07a 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -96,7 +96,8 @@ jobs: ~/.cargo/git seaweed-volume/target seaweed-worker/target/${{ matrix.target }}/release - key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} + seaweed-vector/target/${{ matrix.target }}/release + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }} restore-keys: | rust-docker-${{ matrix.target }}- @@ -137,6 +138,17 @@ jobs: cargo build --release -p weed-lance-worker --target ${{ matrix.target }} cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Build the vector engine + run: | + # An older source_ref predates the engine; its image goes without one. + if [ ! -f seaweed-vector/crates/server/Cargo.toml ]; then + echo "No vector engine in this source ref" + exit 0 + fi + cd seaweed-vector + cargo build --release -p weed-vector-server --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: @@ -145,6 +157,7 @@ jobs: weed-volume-large-disk-${{ matrix.arch }} weed-volume-normal-${{ matrix.arch }} weed-worker-${{ matrix.arch }} + weed-vector-${{ matrix.arch }} build: needs: [setup, build-rust-binaries] @@ -217,8 +230,17 @@ jobs: echo "Placed pre-built Rust worker for ${arch}" fi done + mkdir -p docker/weed-vector-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-vector-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}" + echo "Placed pre-built vector engine for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ ls -la docker/weed-worker-prebuilt/ + ls -la docker/weed-vector-prebuilt/ - name: Docker meta id: docker_meta @@ -342,8 +364,17 @@ jobs: echo "Placed pre-built Rust worker for ${arch}" fi done + mkdir -p docker/weed-vector-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-vector-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}" + echo "Placed pre-built vector engine for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ ls -la docker/weed-worker-prebuilt/ + ls -la docker/weed-vector-prebuilt/ - name: Create BuildKit config for local scan build if: needs.setup.outputs.publish != 'true' run: | diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index c78e5da7dc..47f040fd83 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -43,9 +43,9 @@ concurrency: jobs: # ── Pre-build the Rust binaries natively ──────────────────────────── - # The volume server and the Rust maintenance worker, cross-compiled for - # amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into - # ~15 minutes of native compilation. + # The volume server, the Rust maintenance worker and the vector engine, + # cross-compiled for amd64 and arm64 without QEMU, turning a 5-hour emulated + # cargo build into ~15 minutes of native compilation. build-rust-binaries: runs-on: ubuntu-24.04 strategy: @@ -73,7 +73,8 @@ jobs: ~/.cargo/git seaweed-volume/target seaweed-worker/target/${{ matrix.target }}/release - key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} + seaweed-vector/target/${{ matrix.target }}/release + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }} restore-keys: | rust-docker-${{ matrix.target }}- @@ -114,6 +115,12 @@ jobs: cargo build --release -p weed-lance-worker --target ${{ matrix.target }} cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Build the vector engine + run: | + cd seaweed-vector + cargo build --release -p weed-vector-server --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: @@ -122,6 +129,7 @@ jobs: weed-volume-large-disk-${{ matrix.arch }} weed-volume-normal-${{ matrix.arch }} weed-worker-${{ matrix.arch }} + weed-vector-${{ matrix.arch }} # One job per (variant, platform) on a native runner, pushed by digest; # the merge job stitches the digests into one multi-arch tag. @@ -198,8 +206,17 @@ jobs: echo "Placed pre-built Rust worker for ${arch}" fi done + mkdir -p docker/weed-vector-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-vector-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}" + echo "Placed pre-built vector engine for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ ls -la docker/weed-worker-prebuilt/ + ls -la docker/weed-vector-prebuilt/ - name: Free Disk Space if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant diff --git a/.github/workflows/rust-vector-tests.yml b/.github/workflows/rust-vector-tests.yml index 1776ae513a..431b0d9d99 100644 --- a/.github/workflows/rust-vector-tests.yml +++ b/.github/workflows/rust-vector-tests.yml @@ -68,19 +68,6 @@ jobs: restore-keys: | rust-vector-${{ steps.toolchain.outputs.fingerprint }}- - # lance's build scripts compile their own protos and look for a protoc. - # Point them at the one protoc-bin-vendored ships, so no job depends on a - # system package and every build sees the same version. - - name: Use the vendored protoc - run: | - cd seaweed-vector - cargo fetch - version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock) - test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; } - protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1) - test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; } - echo "PROTOC=$protoc" >> "$GITHUB_ENV" - - name: Build the vector crates run: cd seaweed-vector && cargo build --release @@ -125,15 +112,6 @@ jobs: key: vectors-e2e-${{ runner.os }}-${{ runner.arch }}-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-vector/Cargo.lock', 'seaweed-volume/Cargo.lock') }} restore-keys: | vectors-e2e-${{ runner.os }}-${{ runner.arch }}-${{ steps.toolchain.outputs.fingerprint }}- - - name: Use the vendored protoc - run: | - cd seaweed-vector - cargo fetch - version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock) - test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; } - protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1) - test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; } - echo "PROTOC=$protoc" >> "$GITHUB_ENV" - name: Test Go vector gateway run: | go test ./weed/s3api/s3vectors -count=1 diff --git a/.github/workflows/rust_binaries_release.yml b/.github/workflows/rust_binaries_release.yml index 3999c6df34..f6c6359ac5 100644 --- a/.github/workflows/rust_binaries_release.yml +++ b/.github/workflows/rust_binaries_release.yml @@ -188,6 +188,83 @@ jobs: weed-worker_${{ matrix.asset_suffix }}.tar.gz weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5 + # The vector engine behind the S3 Vectors API: Linux only, like the worker, + # whose lance dependency tree it shares. + build-rust-vector-linux: + permissions: + contents: write + runs-on: ubuntu-24.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-musl + asset_suffix: linux_amd64 + - target: aarch64-unknown-linux-musl + asset_suffix: linux_arm64 + + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up the musl target + uses: ./.github/actions/rust-musl-target + with: + target: ${{ matrix.target }} + + - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-vector/target/${{ matrix.target }}/release + key: rust-vector-release-${{ matrix.target }}-${{ hashFiles('seaweed-vector/Cargo.lock') }} + restore-keys: | + rust-vector-release-${{ matrix.target }}- + + - name: Use the vendored protoc + run: | + cd seaweed-vector + cargo fetch + version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock) + test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; } + protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1) + test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; } + echo "PROTOC=$protoc" >> "$GITHUB_ENV" + + - name: Build the vector engine + run: | + cd seaweed-vector + cargo build --release -p weed-vector-server --target ${{ matrix.target }} + + - name: Package binary + run: | + cp seaweed-vector/target/${{ matrix.target }}/release/weed-vector weed-vector + tar czf weed-vector_${{ matrix.asset_suffix }}.tar.gz weed-vector + rm weed-vector + md5sum weed-vector_${{ matrix.asset_suffix }}.tar.gz > weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5 + + - name: Upload release assets + if: startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v3 + with: + files: | + weed-vector_${{ matrix.asset_suffix }}.tar.gz + weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload artifacts + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + uses: actions/upload-artifact@v7 + with: + name: rust-vector-${{ matrix.asset_suffix }} + path: | + weed-vector_${{ matrix.asset_suffix }}.tar.gz + weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5 + build-rust-volume-darwin: permissions: contents: write diff --git a/docker/Dockerfile.go_build b/docker/Dockerfile.go_build index a2eb20568d..22e4674290 100644 --- a/docker/Dockerfile.go_build +++ b/docker/Dockerfile.go_build @@ -33,6 +33,7 @@ ARG TARGETARCH ARG TAGS COPY weed-volume-prebuilt/ /prebuilt/ COPY weed-worker-prebuilt/ /prebuilt-worker/ +COPY weed-vector-prebuilt/ /prebuilt-vector/ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume # seaweed-common is a path dependency of seaweed-volume that lives beside it, # so the source build below needs it in the same relative position. @@ -65,11 +66,19 @@ RUN if [ -f "/prebuilt-worker/weed-worker-${TARGETARCH}" ]; then \ echo "No pre-built Rust worker for ${TARGETARCH}" && \ touch /weed-worker; \ fi +# The vector engine carries the same lance tree, so it is pre-built or absent too. +RUN if [ -f "/prebuilt-vector/weed-vector-${TARGETARCH}" ]; then \ + echo "Using pre-built vector engine for ${TARGETARCH}" && \ + cp "/prebuilt-vector/weed-vector-${TARGETARCH}" /weed-vector; \ + else \ + echo "No pre-built vector engine for ${TARGETARCH}" && \ + touch /weed-vector; \ + fi # Pre-built binaries arrive via GitHub Actions artifacts, which drop the # executable bit, so the copied file is 0644 and exec fails with "Permission # denied". Restore it (no-op for the empty placeholders, which stay size 0). -RUN chmod 0755 /weed-volume /weed-worker +RUN chmod 0755 /weed-volume /weed-worker /weed-vector FROM alpine AS final LABEL author="Chris Lu" @@ -78,6 +87,8 @@ COPY --from=builder /go/bin/weed /usr/bin/ COPY --from=rust_builder /weed-volume /usr/bin/weed-volume # Same for the Rust maintenance worker, which serves Lance table buckets COPY --from=rust_builder /weed-worker /usr/bin/weed-worker +# And the vector engine behind the S3 Vectors API +COPY --from=rust_builder /weed-vector /usr/bin/weed-vector RUN mkdir -p /etc/seaweedfs COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 0bbd507871..8e6ee840c5 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -100,6 +100,19 @@ case "$1" in exec /usr/bin/weed-worker "$@" ;; + 'vector') + shift + if [ ! -s /usr/bin/weed-vector ]; then + echo "Error: the vector engine is not available on this platform ($(uname -m))." >&2 + exit 1 + fi + # The gateway dials the engine from another container, so not loopback. + if [ -z "$WEED_VECTOR_LISTEN" ] && ! isArgPassed "-listen" "$@"; then + set -- --listen 0.0.0.0:8090 "$@" + fi + exec /usr/bin/weed-vector "$@" + ;; + 'server') ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024" if isArgPassed "-volume.max" "$@"; then diff --git a/docker/weed-vector-prebuilt/.gitkeep b/docker/weed-vector-prebuilt/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/install.sh b/install.sh index 8bede247af..72574bf4cc 100755 --- a/install.sh +++ b/install.sh @@ -9,7 +9,7 @@ # curl -fsSL ... | bash -s -- --version 4.34 --dir /usr/local/bin # # Options: -# --component COMP Which binary to install: weed, volume-rust, worker-rust, all (default: weed) +# --component COMP Which binary to install: weed, volume-rust, worker-rust, vector, all (default: weed) # --version VER Release version tag (default: latest) # --large-disk Use large disk variant (5-byte offset, 8TB max volume) # --dir DIR Installation directory (default: /usr/local/bin) @@ -23,6 +23,7 @@ VERSION="" LARGE_DISK=false INSTALL_DIR="/usr/local/bin" WORKER_INSTALLED=false +VECTOR_INSTALLED=false # Colors (if terminal supports them) if [ -t 1 ]; then @@ -156,6 +157,12 @@ worker_asset_name() { echo "weed-worker_${os}_${arch}.tar.gz" } +# The vector engine behind the S3 Vectors API, built like the worker. +vector_asset_name() { + local os="$1" arch="$2" + echo "weed-vector_${os}_${arch}.tar.gz" +} + # Install a single component install_component() { local component="$1" os="$2" arch="$3" @@ -264,8 +271,38 @@ install_component() { ok "Installed weed-worker to ${INSTALL_DIR}/weed-worker" ;; + vector) + # Published for linux amd64/arm64 only, like the worker. + case "$os" in + linux) ;; + *) error "The vector engine is not available for ${os}. Supported: linux" ;; + esac + case "$arch" in + amd64|arm64) ;; + *) error "The vector engine is not available for ${arch}. Supported: amd64, arm64" ;; + esac + + asset_name="$(vector_asset_name "$os" "$arch")" + download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}" + download "$download_url" "${tmpdir}/${asset_name}" + + info "Extracting ${asset_name}..." + tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir" + + local vector_bin + vector_bin="$(find "$tmpdir" -name 'weed-vector' -type f | head -1)" + if [ -z "$vector_bin" ]; then + error "Could not find weed-vector binary in archive" + fi + + chmod +x "$vector_bin" + install_binary "$vector_bin" "weed-vector" + VECTOR_INSTALLED=true + ok "Installed weed-vector to ${INSTALL_DIR}/weed-vector" + ;; + *) - error "Unknown component: ${component}. Use: weed, volume-rust, worker-rust, all" + error "Unknown component: ${component}. Use: weed, volume-rust, worker-rust, vector, all" ;; esac @@ -327,6 +364,13 @@ main() { else install_component "worker-rust" "$os" "$arch" fi + if [ "$os" != "linux" ] || { [ "$arch" != "amd64" ] && [ "$arch" != "arm64" ]; }; then + warn "Skipping the vector engine: no build for ${os}/${arch}" + elif ! asset_exists "https://github.com/${REPO}/releases/download/${VERSION}/$(vector_asset_name "$os" "$arch")"; then + warn "Skipping the vector engine: ${VERSION} does not carry one" + else + install_component "vector" "$os" "$arch" + fi ;; *) install_component "$COMPONENT" "$os" "$arch" @@ -344,6 +388,9 @@ main() { if [ "$WORKER_INSTALLED" = true ]; then info " weed-worker: ${INSTALL_DIR}/weed-worker" fi + if [ "$VECTOR_INSTALLED" = true ]; then + info " weed-vector: ${INSTALL_DIR}/weed-vector" + fi echo "" info "Quick start:" info " weed master # Start master server" @@ -352,6 +399,9 @@ main() { if [ "$WORKER_INSTALLED" = true ]; then info " weed-worker --admin localhost:12535 # Start the Rust maintenance worker" fi + if [ "$VECTOR_INSTALLED" = true ]; then + info " weed-vector --help # The vector engine for S3 vector buckets" + fi } main diff --git a/k8s/charts/seaweedfs/Chart.yaml b/k8s/charts/seaweedfs/Chart.yaml index 357c1fb96f..68b529b13e 100644 --- a/k8s/charts/seaweedfs/Chart.yaml +++ b/k8s/charts/seaweedfs/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "4.51" +appVersion: "4.52" # Dev note: Trigger a helm chart release by `git tag -a helm-` -version: 4.51.0 +version: 4.52.0 diff --git a/note/star_history.svg b/note/star_history.svg index b7fb4891cc..c82cc2d852 100644 --- a/note/star_history.svg +++ b/note/star_history.svg @@ -6,7 +6,7 @@ - 2026-11-01T00:37:25.935647 + 2026-11-02T00:59:30.216249 image/svg+xml @@ -40,23 +40,23 @@ z - + - - + - + - + - + - + - + - + - + - + - + - + @@ -301,18 +301,18 @@ L 352.912648 26.215385 - + - + - + @@ -325,18 +325,18 @@ L 448.648401 26.215385 - + - + - + @@ -349,18 +349,18 @@ L 544.253188 26.215385 - + - + - + @@ -484,41 +484,41 @@ z - + - - + - + - + - + - + - + - + - + @@ -577,18 +577,18 @@ L 709.2 268.125647 - + - + - + @@ -599,18 +599,18 @@ L 709.2 224.3653 - + - + - + @@ -621,18 +621,18 @@ L 709.2 180.604953 - + - + - + @@ -643,18 +643,18 @@ L 709.2 136.844606 - + - + - + - + - + - + @@ -813,934 +813,937 @@ z +" clip-path="url(#p09ad2cb364)" style="fill: none; stroke: #0969da; stroke-width: 1.6; stroke-linecap: square"/> + diff --git a/seaweed-vector/.cargo/config.toml b/seaweed-vector/.cargo/config.toml new file mode 100644 index 0000000000..d3b5a7e240 --- /dev/null +++ b/seaweed-vector/.cargo/config.toml @@ -0,0 +1,6 @@ +# lance's build scripts run $PROTOC. Point them at the protoc that +# protoc-bin-vendored ships, as this workspace's own build script does, so a +# build needs no system protoc. An explicit PROTOC still wins. Cargo reads this +# file only when it runs inside seaweed-vector/. +[env] +PROTOC = { value = ".cargo/protoc", relative = true } diff --git a/seaweed-vector/.cargo/protoc b/seaweed-vector/.cargo/protoc new file mode 100755 index 0000000000..3f1717c478 --- /dev/null +++ b/seaweed-vector/.cargo/protoc @@ -0,0 +1,21 @@ +#!/bin/sh +# Runs the protoc of the protoc-bin-vendored version in Cargo.lock, or the one +# on PATH when the registry does not hold it. +case "$(uname -s)-$(uname -m)" in +Linux-x86_64) platform=linux-x86_64 ;; +Linux-aarch64) platform=linux-aarch_64 ;; +Linux-ppc64le) platform=linux-ppcle_64 ;; +Linux-s390x) platform=linux-s390_64 ;; +Linux-i?86) platform=linux-x86_32 ;; +Darwin-x86_64) platform=macos-x86_64 ;; +Darwin-arm64) platform=macos-aarch_64 ;; +*) exec protoc "$@" ;; +esac +crate="protoc-bin-vendored-$platform" +version=$(awk -v name="\"$crate\"" '$1 == "name" && $3 == name { found = 1; next } found && $1 == "version" { gsub(/"/, "", $3); print $3; exit }' "$(dirname "$0")/../Cargo.lock") +for protoc in "${CARGO_HOME:-$HOME/.cargo}"/registry/src/*/"$crate-$version"/bin/protoc; do + if [ -x "$protoc" ]; then + exec "$protoc" "$@" + fi +done +exec protoc "$@" diff --git a/seaweed-vector/crates/core/src/filter.rs b/seaweed-vector/crates/core/src/filter.rs index da1fabf458..9613e9cafa 100644 --- a/seaweed-vector/crates/core/src/filter.rs +++ b/seaweed-vector/crates/core/src/filter.rs @@ -1,14 +1,16 @@ //! Lance's json_extract and json_exists parse their JSONPath again for every //! row. A filter's path is a literal, so swap in copies that parse it once per -//! batch and return the same values. +//! batch and return the same values. json_array_contains is swapped too, and +//! compares the elements as typed JSON values: Lance's compares their text, so +//! 1 would match "1". use std::sync::Arc; use arrow_array::builder::{BooleanBuilder, StringBuilder}; use arrow_array::{Array, ArrayRef, LargeBinaryArray, new_null_array}; use arrow_schema::DataType; -use jsonb::RawJsonb; use jsonb::jsonpath::{Selector, parse_json_path}; +use jsonb::{OwnedJsonb, RawJsonb}; use lance::dataset::scanner::Scanner; use lance::deps::datafusion::common::tree_node::{Transformed, TreeNode}; use lance::deps::datafusion::common::{DataFusionError, Result, ScalarValue}; @@ -34,12 +36,16 @@ fn swap(expr: Expr) -> Result { let op = match f.func.name() { "json_extract" => Op::Extract, "json_exists" => Op::Exists, + "json_array_contains" => Op::Contains, _ => return Ok(Transformed::no(Expr::ScalarFunction(f))), }; - if !matches!( - f.args.get(1), - Some(Expr::Literal(ScalarValue::Utf8(Some(_)), _)) - ) { + let literal = |i: usize| { + matches!( + f.args.get(i), + Some(Expr::Literal(ScalarValue::Utf8(Some(_)), _)) + ) + }; + if !literal(1) || (op == Op::Contains && !literal(2)) { return Ok(Transformed::no(Expr::ScalarFunction(f))); } let udf = Arc::new(ScalarUDF::new_from_impl(JsonPath::new(op))); @@ -54,6 +60,7 @@ fn swap(expr: Expr) -> Result { enum Op { Extract, Exists, + Contains, } #[derive(Debug, PartialEq, Eq, Hash)] @@ -64,12 +71,13 @@ struct JsonPath { impl JsonPath { fn new(op: Op) -> Self { + let mut args = vec![DataType::LargeBinary, DataType::Utf8]; + if op == Op::Contains { + args.push(DataType::Utf8); + } Self { op, - signature: Signature::exact( - vec![DataType::LargeBinary, DataType::Utf8], - Volatility::Immutable, - ), + signature: Signature::exact(args, Volatility::Immutable), } } } @@ -83,6 +91,7 @@ impl ScalarUDFImpl for JsonPath { match self.op { Op::Extract => "json_extract", Op::Exists => "json_exists", + Op::Contains => "json_array_contains", } } @@ -93,17 +102,24 @@ impl ScalarUDFImpl for JsonPath { fn return_type(&self, _: &[DataType]) -> Result { Ok(match self.op { Op::Extract => DataType::Utf8, - Op::Exists => DataType::Boolean, + Op::Exists | Op::Contains => DataType::Boolean, }) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let [data, ColumnarValue::Scalar(ScalarValue::Utf8(Some(path)))] = args.args.as_slice() - else { - return Err(exec_err(format!( - "{} expects a literal JSONPath", - self.name() - ))); + let (data, path, value) = match args.args.as_slice() { + [data, ColumnarValue::Scalar(ScalarValue::Utf8(Some(path)))] => (data, path, None), + [ + data, + ColumnarValue::Scalar(ScalarValue::Utf8(Some(path))), + ColumnarValue::Scalar(ScalarValue::Utf8(Some(value))), + ] => (data, path, Some(value)), + _ => { + return Err(exec_err(format!( + "{} expects literal arguments", + self.name() + ))); + } }; let data = data.to_array(args.number_rows)?; let data = data @@ -160,6 +176,37 @@ impl ScalarUDFImpl for JsonPath { } Arc::new(builder.finish()) } + Op::Contains => { + let value = value.map(String::as_str).unwrap_or_default(); + let value: OwnedJsonb = value + .parse() + .map_err(|e| exec_err(format!("Invalid JSON value '{value}': {e}")))?; + let mut builder = BooleanBuilder::with_capacity(data.len()); + for i in 0..data.len() { + if data.is_null(i) { + builder.append_null(); + continue; + } + let mut selector = Selector::new(RawJsonb::new(data.value(i))); + let found = match selector.select_value(&json_path) { + Ok(Some(array)) => array + .as_raw() + .array_values() + .map_err(|e| { + exec_err(format!("Failed to read array at '{path}': {e}")) + })? + .is_some_and(|items| items.contains(&value)), + Ok(None) => false, + Err(e) => { + return Err(exec_err(format!( + "Failed to check array contains at path '{path}': {e}" + ))); + } + }; + builder.append_value(found); + } + Arc::new(builder.finish()) + } }; Ok(ColumnarValue::Array(out)) } @@ -298,6 +345,48 @@ mod tests { assert!(keys(&dataset, bad, true).await.is_err()); } + #[tokio::test] + async fn array_contains_compares_typed_values() { + let dir = TempDir::new().unwrap(); + let metas = [ + Some(r#"{"k": ["space", "classic"]}"#), + Some(r#"{"k": [1, 2.5]}"#), + Some(r#"{"k": ["1", "2.5"]}"#), + Some(r#"{"k": [true]}"#), + Some(r#"{"k": ["true"]}"#), + Some(r#"{"k": "space"}"#), + Some(r#"{"k": [["space"]]}"#), + Some(r#"{"k": {"a": "space"}}"#), + Some(r#"{"k": ["", "spa\"ce"]}"#), + Some(r#"{"k": []}"#), + Some(r#"{"x": 1}"#), + None, + ]; + let dataset = dataset(&dir, &metas).await; + for (value, want) in [ + (r#""space""#, vec!["k0"]), + ("1", vec!["k1"]), + ("1.0", vec!["k1"]), + ("2.5", vec!["k1"]), + (r#""1""#, vec!["k2"]), + ("true", vec!["k3"]), + (r#""true""#, vec!["k4"]), + (r#""""#, vec!["k8"]), + (r#""spa\"ce""#, vec!["k8"]), + (r#""classic""#, vec!["k0"]), + ("3", vec![]), + ] { + let filter = format!(r#"json_array_contains(`metadata`, '$["k"]', '{value}')"#); + assert_eq!( + keys(&dataset, &filter, true).await.unwrap(), + want, + "{filter}" + ); + } + let bad = r#"json_array_contains(`metadata`, '$["k"]', 'not json')"#; + assert!(keys(&dataset, bad, true).await.is_err()); + } + #[tokio::test] async fn apply_swaps_literal_paths_only() { let dir = TempDir::new().unwrap(); @@ -305,7 +394,7 @@ mod tests { let mut scanner = dataset.scan(); apply( &mut scanner, - r#"json_extract(`metadata`, '$["k"]') = '"v"' AND json_exists(`metadata`, '$["k"]') AND json_get_string(`metadata`, 'k') = 'v'"#, + r#"json_extract(`metadata`, '$["k"]') = '"v"' AND json_exists(`metadata`, '$["k"]') AND json_get_string(`metadata`, 'k') = 'v' AND json_array_contains(`metadata`, '$["k"]', '"v"') AND NOT json_array_contains(`metadata`, '$["k"]', key)"#, ) .unwrap(); let mut swapped = Vec::new(); @@ -325,6 +414,8 @@ mod tests { assert_eq!( swapped, [ + ("json_array_contains".to_string(), false), + ("json_array_contains".to_string(), true), ("json_exists".to_string(), true), ("json_extract".to_string(), true), ("json_get_string".to_string(), false), diff --git a/seaweed-vector/crates/server/src/main.rs b/seaweed-vector/crates/server/src/main.rs index 10d46f372d..42bbc9ca3b 100644 --- a/seaweed-vector/crates/server/src/main.rs +++ b/seaweed-vector/crates/server/src/main.rs @@ -1,7 +1,8 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use clap::Parser; use tonic::transport::Server; -use weed_lance_worker::catalog::NamespaceClient; +use tonic::transport::server::TcpIncoming; +use weed_lance_worker::catalog::{NamespaceClient, SigningKey}; use weed_lance_worker::dataset::FallbackOptions; use weed_vector_server::pb::vector::vector_service_server::VectorServiceServer; use weed_vector_server::read_cache::ReadCache; @@ -10,15 +11,15 @@ use weed_vector_server::service::VectorServer; /// The VectorService the S3 gateway fans vector calls out to. #[derive(Parser)] struct Args { - /// Lance Namespace base URL, e.g. http://localhost:8888/lance. + /// Lance Namespace base URL: the S3 gateway's Lance port (-s3.port.lance). #[arg( long, env = "WEED_VECTOR_NAMESPACE", - default_value = "http://localhost:8888/lance" + default_value = "http://localhost:9101" )] namespace: String, /// gRPC listen address. - #[arg(long, env = "WEED_VECTOR_LISTEN", default_value = "127.0.0.1:8080")] + #[arg(long, env = "WEED_VECTOR_LISTEN", default_value = "127.0.0.1:8090")] listen: String, /// object_store option fallback for deployments that vend no credentials: /// key=value, repeatable. @@ -31,6 +32,10 @@ struct Args { /// Memory for caching reads of write-once table files, in MiB; 0 disables. #[arg(long, env = "WEED_VECTOR_READ_CACHE_MB", default_value_t = 512)] read_cache_mb: u64, + /// The `jwt.filer_signing.key` of the cluster. The Lance namespace vends + /// credentials that write index data only to a caller holding it. + #[arg(long, env = "WEED_FILER_SIGNING_KEY")] + filer_signing_key: Option, } fn parse_option(s: &str) -> Result<(String, String), String> { @@ -100,6 +105,11 @@ fn advertised_addr(listen: &str, masters: &[String]) -> String { async fn main() -> Result<()> { tracing_subscriber::fmt::init(); let args = Args::parse_from(normalize_args(std::env::args().collect())); + // Bind before registering, so the master never lists an engine that is + // about to exit because its port is taken. + let incoming = TcpIncoming::bind(args.listen.parse()?) + .with_context(|| format!("listen on {}", args.listen))? + .with_nodelay(Some(true)); if let Some(masters) = &args.masters { let masters: Vec = masters.split(',').map(|s| s.trim().to_string()).collect(); for master in &masters { @@ -111,7 +121,10 @@ async fn main() -> Result<()> { )); } let fallback: FallbackOptions = args.storage_options.into_iter().collect(); - let client = NamespaceClient::new(args.namespace.clone()).with_fallback(&fallback); + let signing_key = SigningKey::new(args.filer_signing_key); + let client = NamespaceClient::new(args.namespace.clone()) + .with_fallback(&fallback) + .with_signing_key(signing_key.as_ref()); let mut service = VectorServer::new(client, fallback); if args.read_cache_mb > 0 { service = service.with_read_cache(ReadCache::new(args.read_cache_mb << 20)); @@ -125,7 +138,7 @@ async fn main() -> Result<()> { .max_decoding_message_size(1 << 30) .max_encoding_message_size(1 << 30), ) - .serve(args.listen.parse()?) + .serve_with_incoming(incoming) .await?; Ok(()) } diff --git a/seaweed-vector/crates/server/src/service.rs b/seaweed-vector/crates/server/src/service.rs index cb9f51b1c4..8ccf041c2b 100644 --- a/seaweed-vector/crates/server/src/service.rs +++ b/seaweed-vector/crates/server/src/service.rs @@ -424,13 +424,13 @@ impl VectorService for VectorServer { if dimension == 0 || dimension > 4096 { return Err(Status::invalid_argument("dimension must be 1..=4096")); } + // The gateway has already created the table through the catalog. let description = self .client - .declare_table(&table.id, None) + .describe_table(&table.id) .await .map_err(internal)?; - let mut options = self.fallback.clone(); - options.extend(description.storage_options.clone()); + let options = dataset::table_options(&description, &self.fallback); let schema = schema_for(dimension, metric, &req.non_filterable_metadata_keys); let empty = RecordBatch::new_empty(schema.clone()); let reader = arrow_array::RecordBatchIterator::new(vec![Ok(empty)].into_iter(), schema); diff --git a/seaweed-volume/src/images.rs b/seaweed-volume/src/images.rs index 9ad7ca71c8..2125f433da 100644 --- a/seaweed-volume/src/images.rs +++ b/seaweed-volume/src/images.rs @@ -143,12 +143,12 @@ fn flip_horizontal_if(img: DynamicImage, do_flip: bool) -> DynamicImage { DynamicImage::ImageRgba8(out) } -/// Returns true if the given MIME type or file path extension indicates a JPEG file. -pub fn is_jpeg(mime_type: &str, path: &str) -> bool { +/// Returns true if the given MIME type or file name extension indicates a JPEG file. +pub fn is_jpeg(mime_type: &str, filename: &str) -> bool { if mime_type == "image/jpeg" { return true; } - let lower = path.to_lowercase(); + let lower = filename.to_lowercase(); lower.ends_with(".jpg") || lower.ends_with(".jpeg") } diff --git a/seaweed-volume/src/remote_storage/s3_tier.rs b/seaweed-volume/src/remote_storage/s3_tier.rs index 1bcea50bb7..98603fb25f 100644 --- a/seaweed-volume/src/remote_storage/s3_tier.rs +++ b/seaweed-volume/src/remote_storage/s3_tier.rs @@ -5,21 +5,153 @@ use std::collections::HashMap; use std::future::Future; +use std::pin::Pin; use std::sync::{Arc, OnceLock, RwLock}; +use std::task::{Context, Poll}; +use std::time::Duration; use aws_sdk_s3::Client; use aws_sdk_s3::config::http::HttpResponse; +use aws_sdk_s3::config::timeout::TimeoutConfig; use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region}; use aws_sdk_s3::error::{DisplayErrorContext, SdkError}; use aws_sdk_s3::operation::get_object::GetObjectError; use aws_sdk_s3::operation::head_object::HeadObjectError; +use aws_sdk_s3::operation::upload_part::UploadPartOutput; +use aws_sdk_s3::operation::upload_part::builders::UploadPartFluentBuilder; +use aws_sdk_s3::primitives::{ByteStream, SdkBody}; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; +use bytes::Bytes; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; -use tokio::sync::Semaphore; +use tokio::sync::{Semaphore, watch}; /// Concurrency limit for multipart upload/download (matches Go's s3manager). const CONCURRENCY: usize = 5; +/// A tier endpoint that drops packets or accepts connections without answering +/// must not hold a read, or a volume server's startup, for minutes per request. +/// Matches Go's s3ConnectTimeout and s3ReadTimeout. The SDK's stalled-stream +/// protection already fails a body that stops arriving. +const S3_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +const S3_READ_TIMEOUT: Duration = Duration::from_secs(60); + +/// The SDK's read timeout runs from the start of the request, so on +/// upload_part it would also cap sending the part body. send_part bounds the +/// response wait instead, from when the body is written, like Go's +/// ResponseHeaderTimeout. +fn upload_part_config() -> aws_sdk_s3::config::Builder { + aws_sdk_s3::config::Builder::default().timeout_config( + TimeoutConfig::builder() + .connect_timeout(S3_CONNECT_TIMEOUT) + .disable_read_timeout() + .build(), + ) +} + +/// A part goes to the connection in chunks this size, so its last chunk is +/// taken only once nearly all of the part has been written. +const UPLOAD_CHUNK_SIZE: usize = 64 * 1024; + +/// The latest attempt to send a part, and whether its body has been sent. +type PartSent = Arc>; + +struct PartBody { + rest: Bytes, + attempt: u64, + sent: PartSent, +} + +impl http_body::Body for PartBody { + type Data = Bytes; + type Error = std::convert::Infallible; + + fn poll_frame( + mut self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + if self.rest.is_empty() { + return Poll::Ready(None); + } + let n = self.rest.len().min(UPLOAD_CHUNK_SIZE); + let chunk = self.rest.split_to(n); + if self.rest.is_empty() { + let attempt = self.attempt; + self.sent.send_if_modified(|state| { + let latest = state.0 == attempt; + state.1 |= latest; + latest + }); + } + Poll::Ready(Some(Ok(http_body::Frame::data(chunk)))) + } + + fn is_end_stream(&self) -> bool { + self.rest.is_empty() + } + + fn size_hint(&self) -> http_body::SizeHint { + http_body::SizeHint::with_exact(self.rest.len() as u64) + } +} + +/// Sends one multipart part, giving up once `response_timeout` passes after +/// the whole part was handed to the connection without a response. A retry +/// rebuilds the body and restarts that clock. The body keeps its in-memory +/// contents, so signing and the checksum header are unchanged. +async fn send_part( + request: UploadPartFluentBuilder, + data: Bytes, + response_timeout: Duration, +) -> Result { + let sent: PartSent = Arc::new(watch::channel((0, false)).0); + let body = { + let sent = sent.clone(); + SdkBody::from(data.clone()).map_preserve_contents(move |_| { + let mut attempt = 0; + sent.send_modify(|state| { + *state = (state.0 + 1, data.is_empty()); + attempt = state.0; + }); + SdkBody::from_body_1_x(PartBody { + rest: data.clone(), + attempt, + sent: sent.clone(), + }) + }) + }; + tokio::select! { + resp = request + .body(ByteStream::new(body)) + .customize() + .config_override(upload_part_config()) + .send() => resp.map_err(|e| DisplayErrorContext(&e).to_string()), + () = wait_unanswered(sent, response_timeout) => Err(format!( + "no response within {response_timeout:?} after sending the part" + )), + } +} + +/// Completes once an attempt's body has been sent and `response_timeout` +/// passes before a later attempt starts. +async fn wait_unanswered(sent: PartSent, response_timeout: Duration) { + let mut state = sent.subscribe(); + loop { + let attempt = state + .wait_for(|&(_, done)| done) + .await + .map_or(0, |state| state.0); + if tokio::time::timeout( + response_timeout, + state.wait_for(|&(latest, _)| latest != attempt), + ) + .await + .is_err() + { + return; + } + } +} + /// A tier transfer failure. The variant is what callers match on; the /// message is the operator-facing text. #[derive(Debug, thiserror::Error)] @@ -107,7 +239,13 @@ impl S3TierBackend { .behavior_version(BehaviorVersion::latest()) .region(Region::new(region.to_string())) .credentials_provider(credentials) - .force_path_style(config.force_path_style); + .force_path_style(config.force_path_style) + .timeout_config( + TimeoutConfig::builder() + .connect_timeout(S3_CONNECT_TIMEOUT) + .read_timeout(S3_READ_TIMEOUT) + .build(), + ); if !config.endpoint.is_empty() { s3_config = s3_config.endpoint_url(&config.endpoint); @@ -231,23 +369,23 @@ impl S3TierBackend { TierError::Io(format!("failed to read file at offset {}: {}", off, e)) })?; - let upload_part_resp = client - .upload_part() - .bucket(&bucket) - .key(&key) - .upload_id(&upload_id) - .part_number(pn) - .body(buf.into()) - .send() - .await - .map_err(|e| { - TierError::Io(format!( - "failed to upload part {} at offset {}: {}", - pn, - off, - DisplayErrorContext(&e) - )) - })?; + let upload_part_resp = send_part( + client + .upload_part() + .bucket(&bucket) + .key(&key) + .upload_id(&upload_id) + .part_number(pn), + buf.into(), + S3_READ_TIMEOUT, + ) + .await + .map_err(|e| { + TierError::Io(format!( + "failed to upload part {} at offset {}: {}", + pn, off, e + )) + })?; let e_tag = upload_part_resp.e_tag().unwrap_or_default().to_string(); @@ -560,6 +698,25 @@ impl S3TierBackend { }) } + /// `read_range_blocking` that gives up once `timeout` passes, returning + /// `Ok(None)`. + pub fn read_range_blocking_within( + self: &Arc, + key: &str, + offset: u64, + size: usize, + timeout: Duration, + ) -> Result>, TierError> { + let backend = Arc::clone(self); + let key = key.to_string(); + block_on_tier_future(async move { + match tokio::time::timeout(timeout, backend.read_range(&key, offset, size)).await { + Ok(read) => read.map(Some), + Err(_) => Ok(None), + } + }) + } + pub fn read_range_blocking( &self, key: &str, @@ -725,6 +882,13 @@ fn describe_join_error(join_error: tokio::task::JoinError) -> TierError { mod tests { use super::*; use crate::remote_storage::s3::tests::{CannedResponse, NO_SUCH_KEY}; + use aws_sdk_s3::config::http::HttpRequest; + use aws_sdk_s3::config::{HttpClient, RuntimeComponents}; + use aws_smithy_runtime_api::client::http::{ + HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector, + }; + use aws_smithy_runtime_api::http::StatusCode; + use http_body::Body as _; use std::collections::HashSet; use tokio::runtime::Handle; @@ -897,6 +1061,235 @@ mod tests { assert!(matches!(err, TierError::Io(_)), "{err:?}"); } + #[test] + fn backend_bounds_connect_and_read_waits() { + let (_listener, backend) = silent_backend(); + let timeouts = backend.client.config().timeout_config().unwrap(); + assert_eq!(timeouts.connect_timeout(), Some(S3_CONNECT_TIMEOUT)); + assert_eq!(timeouts.read_timeout(), Some(S3_READ_TIMEOUT)); + } + + #[derive(Debug)] + struct CaptureTimeouts(Arc>>); + + impl aws_sdk_s3::config::Intercept for CaptureTimeouts { + fn name(&self) -> &'static str { + "CaptureTimeouts" + } + + fn read_before_transmit( + &self, + _context: &aws_sdk_s3::config::interceptors::BeforeTransmitInterceptorContextRef<'_>, + _runtime_components: &aws_sdk_s3::config::RuntimeComponents, + cfg: &mut aws_sdk_s3::config::ConfigBag, + ) -> Result<(), aws_sdk_s3::error::BoxError> { + *self.0.lock().unwrap() = cfg.load::().cloned(); + Err("captured".into()) + } + } + + #[tokio::test] + async fn upload_part_keeps_the_connect_timeout_without_the_read_timeout() { + let (_listener, backend) = silent_backend(); + let seen = Arc::new(std::sync::Mutex::new(None)); + let _ = backend + .client + .upload_part() + .bucket("bucket") + .key("key") + .upload_id("upload") + .part_number(1) + .body(vec![0u8; 8].into()) + .customize() + .config_override(upload_part_config().interceptor(CaptureTimeouts(seen.clone()))) + .send() + .await; + let timeouts = seen + .lock() + .unwrap() + .clone() + .expect("request never reached transmit"); + assert_eq!(timeouts.connect_timeout(), Some(S3_CONNECT_TIMEOUT)); + assert_eq!(timeouts.read_timeout(), None); + } + + fn upload_part_request(backend: &S3TierBackend) -> UploadPartFluentBuilder { + backend + .client + .upload_part() + .bucket("bucket") + .key("key") + .upload_id("upload") + .part_number(1) + } + + #[tokio::test] + async fn send_part_gives_up_when_no_response_follows_the_body() { + let (_listener, backend) = silent_backend(); + let sent = tokio::time::timeout( + Duration::from_secs(30), + send_part( + upload_part_request(&backend), + Bytes::from_static(&[0u8; 8]), + Duration::from_millis(200), + ), + ) + .await + .expect("a part with no response did not give up"); + let err = sent.unwrap_err(); + assert!(err.starts_with("no response within 200ms"), "{err}"); + } + + #[tokio::test(start_paused = true)] + async fn a_retry_restarts_the_response_clock() { + let sent: PartSent = Arc::new(watch::channel((1, true)).0); + let mut unanswered = + std::pin::pin!(wait_unanswered(sent.clone(), Duration::from_millis(200))); + let wait = |ms| tokio::time::sleep(Duration::from_millis(ms)); + tokio::select! { + () = unanswered.as_mut() => panic!("gave up early"), + () = wait(150) => {} + } + // The retry starts and finishes sending before the waiter runs again. + sent.send_replace((2, false)); + sent.send_replace((2, true)); + tokio::select! { + () = unanswered.as_mut() => panic!("kept the first attempt's clock"), + () = wait(150) => {} + } + tokio::select! { + () = unanswered => {} + () = wait(100) => panic!("the retry never timed out"), + } + } + + /// Takes the request body one chunk per `pause`, then answers at once. + #[derive(Debug, Clone)] + struct SlowReader { + pause: Duration, + } + + impl HttpConnector for SlowReader { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + let pause = self.pause; + HttpConnectorFuture::new(async move { + let mut body = std::pin::pin!(request.into_body()); + loop { + tokio::time::sleep(pause).await; + let frame = std::future::poll_fn(|cx| body.as_mut().poll_frame(cx)).await; + if frame.is_none() || body.is_end_stream() { + break; + } + } + let mut response = + HttpResponse::new(StatusCode::try_from(200).unwrap(), SdkBody::empty()); + response.headers_mut().insert("ETag", "\"etag\""); + Ok(response) + }) + } + } + + impl HttpClient for SlowReader { + fn http_connector( + &self, + _settings: &HttpConnectorSettings, + _components: &RuntimeComponents, + ) -> SharedHttpConnector { + SharedHttpConnector::new(self.clone()) + } + } + + #[tokio::test] + async fn send_part_does_not_cap_a_slow_body() { + let config = aws_sdk_s3::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new("us-east-1")) + .credentials_provider(Credentials::new("AKIATEST", "secret", None, None, "test")) + .endpoint_url("http://127.0.0.1:1") + .force_path_style(true) + .http_client(SlowReader { + pause: Duration::from_millis(300), + }) + .build(); + let backend = S3TierBackend { + client: Client::from_conf(config), + bucket: "bucket".to_string(), + storage_class: "STANDARD".to_string(), + }; + let output = send_part( + upload_part_request(&backend), + Bytes::from(vec![0u8; 4 * UPLOAD_CHUNK_SIZE]), + Duration::from_millis(200), + ) + .await + .expect("a part still being sent was cut short"); + assert_eq!(output.e_tag(), Some("\"etag\"")); + } + + #[test] + fn read_range_blocking_within_gives_up_on_a_silent_endpoint() { + let (_listener, backend) = silent_backend(); + let start = std::time::Instant::now(); + let got = backend + .read_range_blocking_within("key", 0, 8, Duration::from_millis(200)) + .unwrap(); + assert!(got.is_none()); + assert!(start.elapsed() < Duration::from_secs(5)); + } + + #[tokio::test] + async fn read_range_gives_up_when_the_body_stalls() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + std::thread::spawn(move || { + let mut held = Vec::new(); + for mut conn in listener.incoming().flatten() { + let mut request = Vec::new(); + let mut byte = [0u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + match std::io::Read::read(&mut conn, &mut byte) { + Ok(1) => request.push(byte[0]), + _ => break, + } + } + let _ = std::io::Write::write_all( + &mut conn, + b"HTTP/1.1 206 Partial Content\r\nContent-Length: 8\r\nContent-Range: bytes 0-7/1024\r\n\r\n", + ); + held.push(conn); + } + }); + let backend = S3TierBackend::new(&S3TierConfig { + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + bucket: "bucket".to_string(), + endpoint, + storage_class: String::new(), + force_path_style: true, + }); + + let read = tokio::time::timeout(Duration::from_secs(30), backend.read_range("key", 0, 8)) + .await + .expect("a read whose body stalls did not give up"); + let err = read.unwrap_err(); + assert!(matches!(err, TierError::Io(_)), "{err:?}"); + } + + fn silent_backend() -> (std::net::TcpListener, Arc) { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let backend = S3TierBackend::new(&S3TierConfig { + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + bucket: "bucket".to_string(), + endpoint: format!("http://{}", listener.local_addr().unwrap()), + storage_class: String::new(), + force_path_style: true, + }); + (listener, Arc::new(backend)) + } + #[test] fn read_range_blocking_no_such_key_is_not_found() { let err = backend_answering(404, NO_SUCH_KEY) diff --git a/seaweed-volume/src/server/handlers.rs b/seaweed-volume/src/server/handlers.rs index a1ceefd012..9fd29faf3e 100644 --- a/seaweed-volume/src/server/handlers.rs +++ b/seaweed-volume/src/server/handlers.rs @@ -733,6 +733,27 @@ pub(crate) async fn check_delete_replicas( Ok(()) } +/// `query` with the fields named in `overrides` replaced by their values. +fn override_query_fields(query: &str, overrides: &[(&str, String)]) -> Result { + let mut fields = Vec::new(); + for field in query.split('&').filter(|field| !field.is_empty()) { + let decoded: Vec<(String, String)> = serde_urlencoded::from_str(field) + .map_err(|e| format!("parse replication query: {e}"))?; + if !decoded + .iter() + .any(|(name, _)| overrides.iter().any(|(o, _)| o == name)) + { + fields.push(field); + } + } + let encoded = serde_urlencoded::to_string(overrides) + .map_err(|e| format!("encode replication query: {e}"))?; + if !encoded.is_empty() { + fields.push(&encoded); + } + Ok(fields.join("&")) +} + /// Helper to synchronously replicate a request to peer volume servers. async fn do_replicated_request( state: &VolumeServerState, @@ -794,16 +815,7 @@ async fn do_replicated_request( return Ok(()); } - let mut query_fields = Vec::new(); - for field in query.split('&').filter(|field| !field.is_empty()) { - let decoded: Vec<(String, String)> = serde_urlencoded::from_str(field) - .map_err(|e| format!("parse replication query: {e}"))?; - if !decoded.iter().any(|(name, _)| name == "type") { - query_fields.push(field); - } - } - query_fields.push("type=replicate"); - let new_query = query_fields.join("&"); + let new_query = override_query_fields(query, &[("type", "replicate".to_string())])?; let mut futures = Vec::new(); for loc in remote_locations { @@ -831,6 +843,12 @@ async fn do_replicated_request( if let Some(auth) = headers.get(axum::http::header::AUTHORIZATION) { req_builder = req_builder.header(axum::http::header::AUTHORIZATION, auth); } + for (name, value) in headers + .iter() + .filter(|(name, _)| name.as_str().starts_with("seaweed-")) + { + req_builder = req_builder.header(name, value); + } if let Some(ref b) = body { req_builder = req_builder.body(b.clone()); @@ -2893,8 +2911,12 @@ pub async fn post_handler( while let Ok(Some(field)) = multipart.next_field().await { let field_name = field.name().map(|s| s.to_string()); let fname = field.file_name().map(clean_windows_path_base); - let fct = field.content_type().map(|m| m.to_string()); let field_headers = field.headers().clone(); + let fct = field_headers + .get(header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); let fce = field_headers .get(header::CONTENT_ENCODING) .and_then(|v| v.to_str().ok()) @@ -3060,15 +3082,18 @@ pub async fn post_handler( let mime_type = if let Some(ref pct) = parsed_content_type { pct.clone() } else { - let multipart_fallback = - if should_parse_multipart && !filename.is_empty() && !is_chunk_manifest { - mime_guess::from_path(&filename) - .first() - .map(|m| m.to_string()) - .unwrap_or_default() - } else { - String::new() - }; + let multipart_fallback = if should_parse_multipart + && !filename.is_empty() + && !is_chunk_manifest + && !is_replicate + { + mime_guess::from_path(&filename) + .first() + .map(|m| m.to_string()) + .unwrap_or_default() + } else { + String::new() + }; headers .get(header::CONTENT_TYPE) .and_then(|v| v.to_str().ok()) @@ -3131,23 +3156,24 @@ pub async fn post_handler( .collect(); // Fix JPEG orientation from EXIF data before storing (matches Go behavior). - let body_data = if state.fix_jpg_orientation && crate::images::is_jpeg(&mime_type, &path) { - crate::images::fix_jpg_orientation(&body_data_raw) - } else { - body_data_raw - }; + let (body_data, body_rewritten) = + if state.fix_jpg_orientation && crate::images::is_jpeg(&mime_type, &filename) { + let fixed = crate::images::fix_jpg_orientation(&body_data_raw); + let rewritten = fixed != body_data_raw; + (fixed, rewritten) + } else { + (body_data_raw, false) + }; // Auto-compress compressible file types (matches Go's IsCompressableFileType). // Only compress if not already gzipped and compression saves >10%. - // Go uses filepath.Base(pu.FileName) for extension detection (not the URL path). - let (final_data, final_is_gzipped) = if !is_gzipped && !is_chunk_manifest { - let ext = { - let dot_pos = filename.rfind('.'); - dot_pos - .map(|p| filename[p..].to_lowercase()) - .unwrap_or_default() - }; - if is_compressible_file_type(&ext, &mime_type) { + // A replica keeps the compression state the primary stored. + let (final_data, final_is_gzipped) = if !is_gzipped && !is_chunk_manifest && !is_replicate { + let ext = filename + .rfind('.') + .map(|p| &filename[p..]) + .unwrap_or_default(); + if is_compressible_file_type(ext, &mime_type) { if let Some(compressed) = try_gzip_data(&body_data) { if compressed.len() * 10 < body_data.len() * 9 { (compressed, true) @@ -3235,23 +3261,55 @@ pub async fn post_handler( }; // The fan-out runs concurrently with the local needle write, like Go's - // ReplicatedWrite: it reads the already-buffered request body, so the two - // sides share no mutable state. The ack still waits for both. + // ReplicatedWrite: replicas get the stored bytes with their compression + // state, so they keep it instead of re-deriving it. The ack waits for both. let replication = if needs_replication { let state_clone = state.clone(); let path_clone = path.clone(); let query_clone = query.clone(); - let headers_clone = headers.clone(); - let body_clone = body.clone(); + let mut overrides = vec![("ts", n.last_modified.to_string())]; + if let Some(t) = ttl.filter(|t| !t.is_empty()) { + overrides.push(("ttl", t.to_string())); + } + if is_chunk_manifest { + overrides.push(("cm", "true".to_string())); + } + if fsync { + overrides.push(("fsync", "true".to_string())); + } + let mut replica_headers = HeaderMap::new(); + if let Some(auth) = headers.get(header::AUTHORIZATION) { + replica_headers.insert(header::AUTHORIZATION, auth.clone()); + } + if n.has_pairs() { + for (key, value) in &pair_map { + if let (Ok(name), Ok(value)) = ( + header::HeaderName::try_from(format!("seaweed-{key}")), + header::HeaderValue::from_str(value), + ) { + replica_headers.insert(name, value); + } + } + } + let (content_type, replica_body) = replica_upload_body( + &filename, + &mime_type, + n.is_compressed().then_some("gzip"), + original_content_md5.as_deref().filter(|_| !body_rewritten), + &n.data, + ); + if let Ok(value) = header::HeaderValue::from_str(&content_type) { + replica_headers.insert(header::CONTENT_TYPE, value); + } Some(tokio::spawn(async move { do_replicated_request( &state_clone, vid.0, Method::POST, &path_clone, - &query_clone, - &headers_clone, - Some(body_clone), + &override_query_fields(&query_clone, &overrides)?, + &replica_headers, + Some(replica_body), ) .await })) @@ -4766,6 +4824,15 @@ fn extract_jwt(headers: &HeaderMap, uri: &axum::http::Uri) -> Option { /// Check if a file type should be compressed based on extension and MIME type. /// Returns true only when we are sure the type is compressible. fn is_compressible_file_type(ext: &str, mtype: &str) -> bool { + let ext = ext.to_ascii_lowercase(); + let ext = ext.as_str(); + let mtype = mtype + .split(';') + .next() + .unwrap_or_default() + .trim() + .to_ascii_lowercase(); + let mtype = mtype.as_str(); // text/* if mtype.starts_with("text/") { return true; @@ -4798,6 +4865,9 @@ fn is_compressible_file_type(ext: &str, mtype: &str) -> bool { if mtype.ends_with("script") { return true; } + if mtype.ends_with("json") || mtype.ends_with("json-seq") { + return true; + } if mtype.ends_with("vnd.rar") { return false; } @@ -4811,6 +4881,41 @@ fn is_compressible_file_type(ext: &str, mtype: &str) -> bool { false } +/// One-part multipart upload of a stored needle, the form Go's upload_content +/// sends: the bytes as stored, labeled with their content encoding. +fn replica_upload_body( + filename: &str, + mime: &str, + content_encoding: Option<&str>, + content_md5: Option<&str>, + data: &[u8], +) -> (String, bytes::Bytes) { + let boundary = uuid::Uuid::new_v4().simple().to_string(); + let mut head = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{}\"\r\n", + filename.replace('\\', "\\\\").replace('"', "\\\"") + ); + if !mime.is_empty() { + head.push_str(&format!("Content-Type: {mime}\r\n")); + } + if let Some(encoding) = content_encoding { + head.push_str(&format!("Content-Encoding: {encoding}\r\n")); + } + if let Some(md5) = content_md5 { + head.push_str(&format!("Content-MD5: {md5}\r\n")); + } + head.push_str("\r\n"); + let tail = format!("\r\n--{boundary}--\r\n"); + let mut body = Vec::with_capacity(head.len() + data.len() + tail.len()); + body.extend_from_slice(head.as_bytes()); + body.extend_from_slice(data); + body.extend_from_slice(tail.as_bytes()); + ( + format!("multipart/form-data; boundary={boundary}"), + body.into(), + ) +} + /// Try to gzip data. Returns None on error. fn try_gzip_data(data: &[u8]) -> Option> { use flate2::Compression; @@ -5283,6 +5388,18 @@ mod tests { assert!(is_compressible_file_type("", "application/javascript")); assert!(!is_compressible_file_type("", "application/zstd")); assert!(!is_compressible_file_type("", "application/vnd.rar")); + assert!(is_compressible_file_type("", "application/json")); + assert!(is_compressible_file_type( + "", + "application/json; charset=utf-8" + )); + assert!(is_compressible_file_type("", "Application/JSON")); + assert!(is_compressible_file_type("", "application/ld+json")); + assert!(is_compressible_file_type("", "application/x-ndjson")); + assert!(is_compressible_file_type("", "application/json-seq")); + assert!(is_compressible_file_type("", "text/plain; charset=utf-8")); + assert!(is_compressible_file_type(".JSON", "")); + assert!(!is_compressible_file_type(".gz", "application/json")); // Audio assert!(is_compressible_file_type(".wav", "audio/wav")); @@ -8245,12 +8362,15 @@ mod tests { ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); *state.current_master_url.write().await = listener.local_addr().unwrap().to_string(); - let master_server = tokio::spawn(async move { axum::serve(listener, master).await.unwrap() }); + let master_server = + tokio::spawn(async move { axum::serve(listener, master).await.unwrap() }); let id = 0x6e7a_0c05u64; let fid = format!("1,{id:x}{TEST_COOKIE:08x}"); let token = gen_jwt(&key, 60, &fid).unwrap(); - let query = "%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&%74ype=client&type=old"; - let expected_query = "%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&type=replicate"; + let query = + "%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&ts=1700000000&%74ype=client&type=old"; + let expected_query = + "%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&ts=1700000000&type=replicate"; let payload = b"authenticated replica bytes"; for method in [Method::POST, Method::DELETE] { let response = tokio::time::timeout( @@ -8299,6 +8419,264 @@ mod tests { peer_server.abort(); } + #[tokio::test] + async fn http_replica_stores_the_primary_needle() { + use crate::storage::super_block::ReplicaPlacement; + use axum::{Json, Router, routing::get}; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = volume_test_state(&tmp); + Arc::get_mut(&mut state).unwrap().self_url = "127.0.0.1:8080".to_string(); + let volume = state + .store + .read() + .unwrap() + .find_volume(VolumeId(1)) + .unwrap() + .1; + volume + .set_replica_placement(ReplicaPlacement::from_string("001").unwrap()) + .unwrap(); + let peer_tmp = tempfile::TempDir::new().unwrap(); + let peer = volume_test_state(&peer_tmp); + let peer_volume = peer + .store + .read() + .unwrap() + .find_volume(VolumeId(1)) + .unwrap() + .1; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let peer_addr = listener.local_addr().unwrap(); + let peer_server = tokio::spawn(async move { + axum::serve( + listener, + super::super::volume_server::build_admin_router(peer), + ) + .await + .unwrap(); + }); + let master = Router::new().route( + "/dir/lookup", + get(move || async move { + Json(serde_json::json!({"locations": [ + {"url": "127.0.0.1:8080"}, + {"url": peer_addr.to_string()} + ]})) + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + *state.current_master_url.write().await = listener.local_addr().unwrap().to_string(); + let master_server = + tokio::spawn(async move { axum::serve(listener, master).await.unwrap() }); + + let payload = "compressible replica text ".repeat(64); + let boundary = "replica-boundary"; + let multipart = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"ttl\"\r\n\r\n3m\r\n\ + --{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"a \\\"b\\\".txt\"\r\n\ + Content-Type: text/plain\r\n\r\n{payload}\r\n--{boundary}--\r\n" + ); + let cases = [ + ( + 0x6e7a_0c06u64, + Request::builder() + .header(header::CONTENT_TYPE, "text/plain") + .header( + header::CONTENT_DISPOSITION, + "attachment; filename=\"notes.txt\"", + ) + .header("Content-MD5", compute_md5_base64(payload.as_bytes())) + .header("Seaweed-Color", "blue"), + Body::from(payload.clone()), + ), + ( + 0x6e7a_0c07u64, + Request::builder().header( + header::CONTENT_TYPE, + format!("multipart/form-data; boundary={boundary}"), + ), + Body::from(multipart), + ), + ( + 0x6e7a_0c0au64, + Request::builder().header( + header::CONTENT_DISPOSITION, + "attachment; filename=\"untyped.txt\"", + ), + Body::from(payload.clone()), + ), + ( + 0x6e7a_0c0bu64, + Request::builder().header( + header::CONTENT_DISPOSITION, + format!("attachment; filename=\"{}.txt\"", "n".repeat(300)), + ), + Body::from(payload.clone()), + ), + ( + 0x6e7a_0c0eu64, + Request::builder() + .header(header::CONTENT_TYPE, "Text/Plain; charset=UTF-8") + .header( + header::CONTENT_DISPOSITION, + "attachment; filename=\"upper.txt\"", + ), + Body::from(payload.clone()), + ), + ]; + for (id, request, body) in cases { + let response = tokio::time::timeout( + std::time::Duration::from_secs(5), + super::super::volume_server::build_admin_router(state.clone()).oneshot( + request + .method(Method::POST) + .uri(format!("/1,{id:x}{TEST_COOKIE:08x}?ts=1700000000")) + .body(body) + .unwrap(), + ), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + let read = |v: &Arc| { + let mut stored = Needle { + id: NeedleId(id), + cookie: Cookie(TEST_COOKIE), + ..Needle::default() + }; + v.read_needle(&mut stored).unwrap(); + stored + }; + let (primary, replica) = (read(&volume), read(&peer_volume)); + assert!(primary.is_compressed()); + assert_eq!( + maybe_decompress_gzip(&primary.data).unwrap(), + payload.as_bytes() + ); + assert_eq!(replica.data, primary.data); + assert_eq!(replica.flags, primary.flags); + assert_eq!(replica.name, primary.name); + assert_eq!(replica.mime, primary.mime); + assert_eq!(replica.pairs, primary.pairs); + assert_eq!(replica.ttl, primary.ttl); + assert_eq!(replica.last_modified, 1700000000); + } + master_server.abort(); + peer_server.abort(); + } + + #[tokio::test] + async fn http_replica_keeps_uncompressed_bytes() { + use tower::ServiceExt; + let tmp = tempfile::TempDir::new().unwrap(); + let state = volume_test_state(&tmp); + let volume = state + .store + .read() + .unwrap() + .find_volume(VolumeId(1)) + .unwrap() + .1; + let payload = "compressible replica text ".repeat(64); + for (id, query, compressed) in [ + (0x6e7a_0c08u64, "?type=replicate", false), + (0x6e7a_0c09u64, "", true), + ] { + let response = super::super::volume_server::build_admin_router(state.clone()) + .oneshot( + Request::builder() + .method(Method::POST) + .uri(format!("/1,{id:x}{TEST_COOKIE:08x}{query}")) + .header(header::CONTENT_TYPE, "text/plain") + .body(Body::from(payload.clone())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + let mut stored = Needle { + id: NeedleId(id), + cookie: Cookie(TEST_COOKIE), + ..Needle::default() + }; + volume.read_needle(&mut stored).unwrap(); + assert_eq!(stored.is_compressed(), compressed, "query {query:?}"); + } + } + + #[tokio::test] + async fn http_orientation_fix_follows_the_filename() { + use tower::ServiceExt; + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = volume_test_state(&tmp); + Arc::get_mut(&mut state).unwrap().fix_jpg_orientation = true; + let volume = state + .store + .read() + .unwrap() + .find_volume(VolumeId(1)) + .unwrap() + .1; + let mut encoded = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgba8(image::RgbaImage::new(3, 2)) + .write_to(&mut encoded, image::ImageFormat::Jpeg) + .unwrap(); + let exif: &[u8] = + b"Exif\0\0MM\0\x2a\0\0\0\x08\0\x01\x01\x12\0\x03\0\0\0\x01\0\x03\0\0\0\0\0\0"; + let mut tagged = vec![0xFF, 0xD8, 0xFF, 0xE1, 0, exif.len() as u8 + 2]; + tagged.extend_from_slice(exif); + tagged.extend_from_slice(&encoded.into_inner()[2..]); + let upload = |id: u64, suffix: &'static str, filename: &'static str, body: Vec| { + let (state, volume) = (state.clone(), volume.clone()); + async move { + let response = super::super::volume_server::build_admin_router(state) + .oneshot( + Request::builder() + .method(Method::POST) + .uri(format!("/1,{id:x}{TEST_COOKIE:08x}{suffix}")) + .header(header::CONTENT_TYPE, "application/octet-stream") + .header( + header::CONTENT_DISPOSITION, + format!("attachment; filename=\"{filename}\""), + ) + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::CREATED, + "{suffix} {filename}" + ); + let mut stored = Needle { + id: NeedleId(id), + cookie: Cookie(TEST_COOKIE), + ..Needle::default() + }; + volume.read_needle(&mut stored).unwrap(); + stored.data + } + }; + let rotated = upload(0x6e7a_0c10, "", "photo.jpg", tagged.clone()).await; + assert_ne!(rotated, tagged); + assert_eq!( + upload(0x6e7a_0c11, ".jpg", "photo.png", tagged.clone()).await, + tagged + ); + assert_eq!( + upload(0x6e7a_0c12, "?type=replicate", "photo.jpg", tagged.clone()).await, + rotated + ); + assert_eq!( + upload(0x6e7a_0c13, "?type=replicate", "photo.jpg", rotated.clone()).await, + rotated + ); + } + #[tokio::test] async fn http_multipart_replica_refusal_is_not_marked() { use tower::ServiceExt; diff --git a/seaweed-volume/src/storage/volume.rs b/seaweed-volume/src/storage/volume.rs index 369eb44e75..09fdb62967 100644 --- a/seaweed-volume/src/storage/volume.rs +++ b/seaweed-volume/src/storage/volume.rs @@ -9,14 +9,14 @@ //! Matches Go's storage/volume.go, volume_loading.go, volume_read.go, //! volume_write.go, volume_super_block.go. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fs::{self, File, OpenOptions}; use std::io::{self, Read, Seek, SeekFrom, Write}; use std::ops::ControlFlow; use std::path::Path; use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU8, AtomicU64, Ordering}; -use std::sync::{Arc, Condvar, Mutex, RwLock}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::sync::{Arc, Condvar, LazyLock, Mutex, RwLock}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tracing::{debug, error, info, warn}; @@ -1198,6 +1198,33 @@ impl RemoteDatFile { .backend .read_range_blocking(&self.key, offset, buf.len()) .map_err(io::Error::other)?; + Self::fill(buf, offset, data) + } + + /// `read_exact_at` that fails with `TimedOut` once `timeout` passes. + pub(crate) fn read_exact_at_within( + &self, + buf: &mut [u8], + offset: u64, + timeout: Duration, + ) -> io::Result<()> { + match self + .backend + .read_range_blocking_within(&self.key, offset, buf.len(), timeout) + .map_err(io::Error::other)? + { + Some(data) => Self::fill(buf, offset, data), + None => Err(io::Error::new( + io::ErrorKind::TimedOut, + format!( + "remote read at offset {} got no answer within {:?}", + offset, timeout + ), + )), + } + } + + fn fill(buf: &mut [u8], offset: u64, data: Vec) -> io::Result<()> { if data.len() != buf.len() { return Err(io::Error::new( io::ErrorKind::UnexpectedEof, @@ -1214,6 +1241,71 @@ impl RemoteDatFile { } } +/// A tiered volume loads without its superblock when the backend cannot serve +/// it; once a read stalls, the backend's other volumes skip it for a while. +/// Matches Go's remoteSuperBlockTimeout and remoteSuperBlockStallPeriod. +const REMOTE_SUPER_BLOCK_TIMEOUT: Duration = Duration::from_secs(10); +const REMOTE_SUPER_BLOCK_STALL_PERIOD: Duration = Duration::from_secs(60); +static REMOTE_SUPER_BLOCK_STALLS: LazyLock>> = + LazyLock::new(Default::default); + +fn read_super_block_bytes( + mut read: impl FnMut(&mut [u8], u64) -> Result<(), VolumeError>, +) -> Result, VolumeError> { + let mut header = [0u8; SUPER_BLOCK_SIZE]; + read(&mut header, 0)?; + + let extra_size = u16::from_be_bytes([header[6], header[7]]); + let mut full_buf = vec![0u8; SUPER_BLOCK_SIZE + extra_size as usize]; + full_buf[..SUPER_BLOCK_SIZE].copy_from_slice(&header); + if extra_size > 0 { + read(&mut full_buf[SUPER_BLOCK_SIZE..], SUPER_BLOCK_SIZE as u64)?; + } + Ok(full_buf) +} + +fn read_remote_super_block( + storage_name: &str, + remote: &RemoteDatFile, + timeout: Duration, + stall_period: Duration, +) -> Result, VolumeError> { + if let Some(until) = REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().get(storage_name) + && Instant::now() < *until + { + return Err(VolumeError::Io(io::Error::other(format!( + "backend {} stalled on a super block read, skipped", + storage_name + )))); + } + let deadline = Instant::now() + timeout; + let result = read_super_block_bytes(|buf, offset| { + remote + .read_exact_at_within( + buf, + offset, + deadline.saturating_duration_since(Instant::now()), + ) + .map_err(VolumeError::from) + }); + match &result { + Ok(_) => { + REMOTE_SUPER_BLOCK_STALLS + .lock() + .unwrap() + .remove(storage_name); + } + Err(VolumeError::Io(e)) if e.kind() == io::ErrorKind::TimedOut => { + REMOTE_SUPER_BLOCK_STALLS + .lock() + .unwrap() + .insert(storage_name.to_string(), Instant::now() + stall_period); + } + Err(_) => {} + } + result +} + // ============================================================================ // Volume // ============================================================================ @@ -1684,6 +1776,7 @@ impl Volume { ))); } + let mut remote_super_block_failed = false; if already_has_super_block { match self.read_super_block(io) { Ok(()) => { @@ -1700,6 +1793,12 @@ impl Volume { error = %e, "failed to read remote super block during load" ); + remote_super_block_failed = true; + let replication = self.volume_info.read().unwrap().replication.clone(); + if !replication.is_empty() { + self.super_block.write().unwrap().replica_placement = + ReplicaPlacement::from_string(&replication)?; + } } Err(e) => return Err(e), } @@ -1748,17 +1847,24 @@ impl Volume { // at the front of .idx. Best effort: a volume that cannot be // repaired is still servable for everything the surviving rows // index. - match self.repair_idx_head_tombstones(io) { - Ok(0) => {} - Ok(restored) => info!( + if remote_super_block_failed { + debug!( volume_id = self.id.0, - restored, "recovered overwritten .idx rows from .dat" - ), - Err(e) => warn!( - volume_id = self.id.0, - error = %e, - "recover overwritten .idx rows" - ), + "remote backend unavailable, not recovering overwritten .idx rows on this load" + ); + } else { + match self.repair_idx_head_tombstones(io) { + Ok(0) => {} + Ok(restored) => info!( + volume_id = self.id.0, + restored, "recovered overwritten .idx rows from .dat" + ), + Err(e) => warn!( + volume_id = self.id.0, + error = %e, + "recover overwritten .idx rows" + ), + } } // The permission bits still allow writing on a read-only mount; @@ -2220,21 +2326,15 @@ impl Volume { // ---- SuperBlock I/O ---- fn read_super_block(&self, io: &VolumeIo) -> Result<(), VolumeError> { - let mut header = [0u8; SUPER_BLOCK_SIZE]; - self.read_exact_at_backend(io, &mut header, 0)?; - - let extra_size = u16::from_be_bytes([header[6], header[7]]); - let total_size = SUPER_BLOCK_SIZE + extra_size as usize; - - let mut full_buf = vec![0u8; total_size]; - full_buf[..SUPER_BLOCK_SIZE].copy_from_slice(&header); - if extra_size > 0 { - self.read_exact_at_backend( - io, - &mut full_buf[SUPER_BLOCK_SIZE..], - SUPER_BLOCK_SIZE as u64, - )?; - } + let full_buf = match io.remote_dat_file.as_ref() { + Some(remote) if self.has_remote_file() => read_remote_super_block( + &self.remote_storage_name_key().0, + remote, + REMOTE_SUPER_BLOCK_TIMEOUT, + REMOTE_SUPER_BLOCK_STALL_PERIOD, + )?, + _ => read_super_block_bytes(|buf, offset| self.read_exact_at_backend(io, buf, offset))?, + }; *self.super_block.write().unwrap() = SuperBlock::from_bytes(&full_buf)?; @@ -11523,6 +11623,254 @@ mod tests { assert!(result.is_err(), "unopenable .dat must fail load"); } + fn silent_tier_backend() -> ( + std::net::TcpListener, + Arc, + ) { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let backend = crate::remote_storage::s3_tier::S3TierBackend::new( + &crate::remote_storage::s3_tier::S3TierConfig { + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + bucket: "bucket-a".to_string(), + endpoint: format!("http://{}", listener.local_addr().unwrap()), + storage_class: "STANDARD".to_string(), + force_path_style: true, + }, + ); + (listener, Arc::new(backend)) + } + + #[test] + fn remote_super_block_read_gives_up_on_a_silent_backend_and_skips_it() { + let name = "s3.super_block_stall_test"; + let (_listener, backend) = silent_tier_backend(); + let silent = RemoteDatFile { + backend, + key: "remote-key".to_string(), + file_size: 1024, + modified_time: 0, + }; + + let start = Instant::now(); + let err = read_remote_super_block( + name, + &silent, + Duration::from_millis(200), + Duration::from_secs(3600), + ) + .unwrap_err(); + assert!( + matches!(&err, VolumeError::Io(e) if e.kind() == io::ErrorKind::TimedOut), + "{err:?}" + ); + assert!(start.elapsed() < Duration::from_secs(5)); + + let start = Instant::now(); + assert!( + read_remote_super_block( + name, + &silent, + Duration::from_secs(30), + Duration::from_secs(3600) + ) + .is_err() + ); + assert!( + start.elapsed() < Duration::from_secs(1), + "a stalled backend must not be asked again within the stall period" + ); + + let sb = SuperBlock { + version: Version::current(), + ..SuperBlock::default() + }; + let (endpoint, shutdown_tx) = spawn_fake_s3_server(sb.to_bytes()); + let answering = RemoteDatFile { + backend: Arc::new(crate::remote_storage::s3_tier::S3TierBackend::new( + &crate::remote_storage::s3_tier::S3TierConfig { + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + bucket: "bucket-a".to_string(), + endpoint, + storage_class: "STANDARD".to_string(), + force_path_style: true, + }, + )), + key: "remote-key".to_string(), + file_size: SUPER_BLOCK_SIZE as u64, + modified_time: 0, + }; + REMOTE_SUPER_BLOCK_STALLS + .lock() + .unwrap() + .insert(name.to_string(), Instant::now()); + let bytes = read_remote_super_block( + name, + &answering, + Duration::from_secs(10), + Duration::from_secs(3600), + ) + .unwrap(); + assert_eq!( + SuperBlock::from_bytes(&bytes).unwrap().version, + Version::current() + ); + assert!( + !REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().contains_key(name), + "a successful read clears the stall" + ); + let _ = shutdown_tx.send(()); + } + + #[test] + fn remote_volume_loads_without_waiting_on_a_stalled_backend() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let dat_len = { + let v = make_test_volume(dir); + v.dat_file_size().unwrap() + }; + std::fs::remove_file(format!("{}/1.dat", dir)).unwrap(); + + let (_listener, backend) = silent_tier_backend(); + let name = "s3.load_stall_test"; + crate::remote_storage::s3_tier::global_s3_tier_registry() + .write() + .unwrap() + .register(name.to_string(), Arc::try_unwrap(backend).ok().unwrap()); + REMOTE_SUPER_BLOCK_STALLS + .lock() + .unwrap() + .insert(name.to_string(), Instant::now() + Duration::from_secs(3600)); + + let vif = VifVolumeInfo { + files: vec![VifRemoteFile { + backend_type: "s3".to_string(), + backend_id: "load_stall_test".to_string(), + key: "remote-key".to_string(), + offset: 0, + file_size: dat_len, + modified_time: 123, + extension: ".dat".to_string(), + }], + version: Version::current().0 as u32, + replication: "001".to_string(), + ..VifVolumeInfo::default() + }; + std::fs::write( + format!("{}/1.vif", dir), + serde_json::to_string_pretty(&vif).unwrap(), + ) + .unwrap(); + + let start = Instant::now(); + let v = Volume::new( + dir, + dir, + VolumeId(1), + NeedleMapKind::InMemory, + &VolumeSpec::default(), + ) + .unwrap(); + assert!(start.elapsed() < Duration::from_secs(5)); + assert!(v.has_remote_file()); + assert_eq!(v.version(), Version::current()); + assert_eq!( + v.super_block.read().unwrap().replica_placement, + ReplicaPlacement::from_string("001").unwrap() + ); + + crate::remote_storage::s3_tier::global_s3_tier_registry() + .write() + .unwrap() + .remove(name); + REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().remove(name); + } + + #[test] + fn remote_volume_load_skips_idx_repair_on_a_stalled_backend() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let dat_len = { + let v = make_test_volume(dir); + for i in 1..=3u64 { + let mut n = Needle { + id: NeedleId(i), + cookie: Cookie(0x1234), + data: format!("needle-{i}").into_bytes(), + data_size: format!("needle-{i}").len() as u32, + ..Needle::default() + }; + v.write_needle(&mut n, true, false).unwrap(); + } + v.dat_file_size().unwrap() + }; + std::fs::remove_file(format!("{}/1.dat", dir)).unwrap(); + let idx_path = format!("{}/1.idx", dir); + crate::storage::volume_idx_repair::tests::clobber_idx_head(&idx_path, &[4, 5]); + let idx_before = std::fs::read(&idx_path).unwrap(); + + let (_listener, backend) = silent_tier_backend(); + let name = "s3.idx_repair_stall_test"; + crate::remote_storage::s3_tier::global_s3_tier_registry() + .write() + .unwrap() + .register(name.to_string(), Arc::try_unwrap(backend).ok().unwrap()); + REMOTE_SUPER_BLOCK_STALLS + .lock() + .unwrap() + .insert(name.to_string(), Instant::now() + Duration::from_secs(3600)); + + let vif = VifVolumeInfo { + files: vec![VifRemoteFile { + backend_type: "s3".to_string(), + backend_id: "idx_repair_stall_test".to_string(), + key: "remote-key".to_string(), + offset: 0, + file_size: dat_len, + modified_time: 123, + extension: ".dat".to_string(), + }], + version: Version::current().0 as u32, + ..VifVolumeInfo::default() + }; + std::fs::write( + format!("{}/1.vif", dir), + serde_json::to_string_pretty(&vif).unwrap(), + ) + .unwrap(); + + let start = Instant::now(); + let v = Volume::new( + dir, + dir, + VolumeId(1), + NeedleMapKind::InMemory, + &VolumeSpec::default(), + ) + .unwrap(); + assert!( + start.elapsed() < Duration::from_secs(5), + "the load waited on the stalled backend" + ); + assert!(v.has_remote_file()); + drop(v); + assert_eq!( + std::fs::read(&idx_path).unwrap(), + idx_before, + "the repair ran against an unreachable backend" + ); + + crate::remote_storage::s3_tier::global_s3_tier_registry() + .write() + .unwrap() + .remove(name); + REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().remove(name); + } + #[test] fn test_remote_only_volume_load_reads_from_tier_backend() { let tmp = TempDir::new().unwrap(); diff --git a/seaweed-volume/src/storage/volume_idx_repair.rs b/seaweed-volume/src/storage/volume_idx_repair.rs index e79e222633..0fd410db8a 100644 --- a/seaweed-volume/src/storage/volume_idx_repair.rs +++ b/seaweed-volume/src/storage/volume_idx_repair.rs @@ -224,7 +224,7 @@ fn finish_walk(walk: io::Result<()>) -> Result<(), VolumeError> { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use crate::storage::needle::crc::CRC; use crate::storage::needle_map::NeedleMapKind; @@ -267,7 +267,7 @@ mod tests { /// Reproduce the damage a delete on a tiered read-only volume used to do: it /// writes (key, offset 0, tombstone) rows over the front of .idx instead of /// appending them. - fn clobber_idx_head(idx_path: &str, keys: &[u64]) { + pub(crate) fn clobber_idx_head(idx_path: &str, keys: &[u64]) { let mut file = OpenOptions::new().write(true).open(idx_path).unwrap(); for (i, key) in keys.iter().enumerate() { let mut row = Vec::new(); diff --git a/seaweed-worker/README.md b/seaweed-worker/README.md index 9dde317711..02497807d9 100644 --- a/seaweed-worker/README.md +++ b/seaweed-worker/README.md @@ -95,6 +95,11 @@ gateway without STS configured vends no credentials at all, so `--access-key` and `--secret-key` supply a fallback; anything the namespace does vend wins over them. +In a vector bucket's `vector` namespace the gateway vends write credentials only +to a caller that proves it holds `jwt.filer_signing.key`. Pass that key with +`--filer-signing-key` so compaction, cleanup and index builds can rewrite +vector indexes. + ## State All three jobs are implemented and tested end to end against a live gateway: diff --git a/seaweed-worker/crates/core/src/token.rs b/seaweed-worker/crates/core/src/token.rs index 3df69398ef..21c01b9e08 100644 --- a/seaweed-worker/crates/core/src/token.rs +++ b/seaweed-worker/crates/core/src/token.rs @@ -33,9 +33,26 @@ pub fn admin_token(signing_key: &[u8], expires_after_secs: u64) -> Option Vec { + let key = hmac::Key::new(hmac::HMAC_SHA256, signing_key); + hmac::sign(&key, purpose.as_bytes()).as_ref().to_vec() +} + #[cfg(test)] mod tests { - use super::admin_token; + use super::{admin_token, derive_key}; + + #[test] + fn derive_key_matches_the_go_derivation() { + let derived = derive_key(b"cluster-key", "seaweedfs-lance-cluster-token"); + let hex: String = derived.iter().map(|b| format!("{b:02x}")).collect(); + assert_eq!( + hex, + "b38c0b66da41621f2a9b6de68e4239ddd0d2dd7b4ede01aa91bd34a17edc1021" + ); + } #[test] fn no_key_no_token() { diff --git a/seaweed-worker/crates/lance/src/catalog/mod.rs b/seaweed-worker/crates/lance/src/catalog/mod.rs index 142de4eb52..a3a91ad307 100644 --- a/seaweed-worker/crates/lance/src/catalog/mod.rs +++ b/seaweed-worker/crates/lance/src/catalog/mod.rs @@ -7,4 +7,4 @@ pub mod namespace; -pub use namespace::{NamespaceClient, TableDescription, parse_id}; +pub use namespace::{NamespaceClient, SigningKey, TableDescription, parse_id}; diff --git a/seaweed-worker/crates/lance/src/catalog/namespace.rs b/seaweed-worker/crates/lance/src/catalog/namespace.rs index 6aca480495..bbbfda6f0f 100644 --- a/seaweed-worker/crates/lance/src/catalog/namespace.rs +++ b/seaweed-worker/crates/lance/src/catalog/namespace.rs @@ -8,6 +8,39 @@ use serde::Deserialize; /// The delimiter the Lance namespace joins identifier parts with. const DELIMITER: &str = "$"; +/// Carries a token that proves the caller holds the cluster key, which marks +/// it as one of the cluster's own components. +const CLUSTER_TOKEN_HEADER: &str = "X-Seaweedfs-Cluster-Token"; + +/// A cluster token is minted for each request, so it only has to outlast the +/// clock skew between this host and the gateway. +const CLUSTER_TOKEN_SECS: u64 = 300; + +/// The key cluster tokens are signed with, derived from the cluster key +/// (`jwt.filer_signing.key`) the way the gateway derives it, so a token +/// captured on the wire does not authorize admin calls. The catalog vends +/// credentials that write a vector bucket's index data only to callers that +/// send one. Not Debug, so the key cannot reach a log. +#[derive(Clone)] +pub struct SigningKey { + key: Vec, +} + +impl SigningKey { + pub fn new(key: Option) -> Option { + key.filter(|key| !key.is_empty()).map(|key| Self { + key: seaweed_worker_core::token::derive_key( + key.as_bytes(), + "seaweedfs-lance-cluster-token", + ), + }) + } + + fn token(&self) -> Option { + seaweed_worker_core::token::admin_token(&self.key, CLUSTER_TOKEN_SECS) + } +} + #[derive(Debug, Deserialize)] pub struct TableDescription { pub location: String, @@ -35,6 +68,7 @@ pub struct NamespaceClient { base_url: String, http: reqwest::Client, api_key: Option, + signing_key: Option, } /// A namespace call that has not answered by now is not going to. Without this @@ -71,6 +105,7 @@ impl NamespaceClient { base_url: base_url.into().trim_end_matches('/').to_string(), http, api_key: None, + signing_key: None, } } @@ -92,10 +127,25 @@ impl NamespaceClient { self } + pub fn with_signing_key(mut self, signing_key: Option<&SigningKey>) -> Self { + if signing_key.is_some() && !protected_transport(&self.base_url) { + tracing::warn!( + "catalog url {} exposes the cluster token in cleartext", + self.base_url + ); + } + self.signing_key = signing_key.cloned(); + self + } + fn authed(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder { - match &self.api_key { + let request = match &self.api_key { Some(key) => request.header("x-api-key", key), None => request, + }; + match self.signing_key.as_ref().and_then(SigningKey::token) { + Some(token) => request.header(CLUSTER_TOKEN_HEADER, token), + None => request, } } @@ -224,7 +274,74 @@ pub fn parse_id(encoded: &str) -> Vec { #[cfg(test)] mod tests { - use super::{MAX_REDIRECT_HOPS, may_follow, protected_transport, same_origin}; + use super::{ + CLUSTER_TOKEN_SECS, MAX_REDIRECT_HOPS, NamespaceClient, SigningKey, may_follow, + protected_transport, same_origin, + }; + + /// A catalog that answers one request with a table location and hands back + /// the request it read. + async fn recording_catalog() -> (String, tokio::sync::oneshot::Receiver) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let (sender, receiver) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + let (mut conn, _) = listener.accept().await.unwrap(); + let mut request = vec![0u8; 8192]; + let n = conn.read(&mut request).await.unwrap(); + let _ = sender.send(String::from_utf8_lossy(&request[..n]).into_owned()); + let body = r#"{"location":"s3://b/vector/t"}"#; + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len() + ); + let _ = conn.write_all(response.as_bytes()).await; + }); + (url, receiver) + } + + fn header<'a>(request: &'a str, name: &str) -> Option<&'a str> { + request.lines().find_map(|line| { + let (key, value) = line.split_once(':')?; + key.eq_ignore_ascii_case(name).then(|| value.trim()) + }) + } + + #[tokio::test] + async fn signing_key_sends_a_cluster_token() { + let id = ["b".to_string(), "vector".to_string(), "t".to_string()]; + let (url, request) = recording_catalog().await; + let key = SigningKey::new(Some("cluster-key".into())); + NamespaceClient::new(url) + .with_signing_key(key.as_ref()) + .describe_table(&id) + .await + .unwrap(); + let request = request.await.unwrap(); + let token = header(&request, "x-seaweedfs-cluster-token").expect("cluster token"); + let raw = + seaweed_worker_core::token::admin_token(b"cluster-key", CLUSTER_TOKEN_SECS).unwrap(); + assert_eq!(token.split('.').count(), 3); + assert_ne!( + token.rsplit('.').next(), + raw.rsplit('.').next(), + "signed with the cluster key itself" + ); + + let (url, request) = recording_catalog().await; + NamespaceClient::new(url).describe_table(&id).await.unwrap(); + assert_eq!( + header(&request.await.unwrap(), "x-seaweedfs-cluster-token"), + None + ); + } + + #[test] + fn an_empty_signing_key_is_no_key() { + assert!(SigningKey::new(None).is_none()); + assert!(SigningKey::new(Some(String::new())).is_none()); + } #[test] fn may_follow_bounds_hops_within_the_origin() { diff --git a/seaweed-worker/crates/lance/src/dataset.rs b/seaweed-worker/crates/lance/src/dataset.rs index d28bcf2d4c..cdca46d0bd 100644 --- a/seaweed-worker/crates/lance/src/dataset.rs +++ b/seaweed-worker/crates/lance/src/dataset.rs @@ -65,21 +65,38 @@ pub async fn open_with( }) } -/// The namespace vends object_store's own option names, so they pass straight -/// through. What it vends always wins: the fallback exists because a deployment -/// without STS vends no credentials at all, and then the worker has no other way -/// to reach the data. +/// The object_store options for a table. The namespace vends object_store's own +/// option names, so they pass straight through. What it vends always wins: the +/// fallback exists because a deployment without STS vends no credentials at +/// all, and then the worker has no other way to reach the data. +pub fn table_options( + description: &TableDescription, + fallback: &FallbackOptions, +) -> HashMap { + let mut options: HashMap = fallback.clone(); + options.extend(description.storage_options.clone()); + // A store that takes unsigned requests needs no credentials, but without + // any lance builds the AWS default chain, and that probes instance + // metadata on every open. + if options + .get("aws_skip_signature") + .is_some_and(|v| v == "true") + { + options.entry("aws_access_key_id".into()).or_default(); + options.entry("aws_secret_access_key".into()).or_default(); + } + options +} + async fn open_at( description: &TableDescription, fallback: &FallbackOptions, wrapper: Option>, ) -> Result { - let mut options: HashMap = fallback.clone(); - options.extend(description.storage_options.clone()); DatasetBuilder::from_uri(&description.location) .with_store_params(ObjectStoreParams { storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( - options, + table_options(description, fallback), ))), object_store_wrapper: wrapper, ..Default::default() @@ -102,14 +119,12 @@ pub async fn drop( Err(err) => return Err(err), }; check_drop_location(id, &description.location)?; - let mut options: HashMap = fallback.clone(); - options.extend(description.storage_options.clone()); let (store, path) = ObjectStore::from_uri_and_params( Arc::new(ObjectStoreRegistry::default()), &description.location, &ObjectStoreParams { storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( - options, + table_options(&description, fallback), ))), ..Default::default() }, @@ -203,12 +218,45 @@ fn schema_json(dataset: &Dataset) -> Option { #[cfg(test)] mod tests { - use super::{check_drop_location, table_location}; + use super::{FallbackOptions, check_drop_location, table_location, table_options}; + use crate::catalog::TableDescription; fn id(parts: &[&str]) -> Vec { parts.iter().map(|part| part.to_string()).collect() } + fn options(pairs: &[(&str, &str)]) -> FallbackOptions { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + #[test] + fn table_options_keep_an_unsigned_store_off_the_aws_default_chain() { + let described = |vended: &[(&str, &str)]| TableDescription { + location: "s3://bkt/vector/x".into(), + storage_options: options(vended), + managed_versioning: false, + }; + let unsigned = described(&[("aws_skip_signature", "true")]); + + let got = table_options(&unsigned, &FallbackOptions::new()); + assert_eq!(got.get("aws_access_key_id").map(String::as_str), Some("")); + assert_eq!( + got.get("aws_secret_access_key").map(String::as_str), + Some("") + ); + + let fallback = options(&[("aws_access_key_id", "k"), ("aws_secret_access_key", "s")]); + let got = table_options(&unsigned, &fallback); + assert_eq!(got.get("aws_access_key_id").map(String::as_str), Some("k")); + + let signed = described(&[("aws_endpoint", "http://s3:8333")]); + let got = table_options(&signed, &FallbackOptions::new()); + assert!(!got.contains_key("aws_access_key_id"), "{got:?}"); + } + #[test] fn table_location_matches_the_gateway_layout() { assert_eq!( diff --git a/seaweed-worker/crates/lance/src/jobs/cleanup.rs b/seaweed-worker/crates/lance/src/jobs/cleanup.rs index c8c4a3ec36..028739d688 100644 --- a/seaweed-worker/crates/lance/src/jobs/cleanup.rs +++ b/seaweed-worker/crates/lance/src/jobs/cleanup.rs @@ -13,7 +13,7 @@ use seaweed_worker_core::pb::{ use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler}; use tracing::warn; -use crate::catalog::{NamespaceClient, parse_id}; +use crate::catalog::{NamespaceClient, SigningKey, parse_id}; use crate::dataset; use crate::jobs::{clamp, string_list, table_id}; @@ -32,6 +32,7 @@ const MAX_MIN_VERSIONS: i64 = 1000; pub struct CleanupVersionsHandler { namespace_url: String, fallback: dataset::FallbackOptions, + signing_key: Option, metrics: Option, } @@ -40,6 +41,7 @@ impl CleanupVersionsHandler { Self { namespace_url, fallback: dataset::FallbackOptions::new(), + signing_key: None, metrics: None, } } @@ -54,8 +56,15 @@ impl CleanupVersionsHandler { self } + pub fn with_signing_key(mut self, signing_key: Option) -> Self { + self.signing_key = signing_key; + self + } + fn client(&self) -> NamespaceClient { - NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback) + NamespaceClient::new(self.namespace_url.clone()) + .with_fallback(&self.fallback) + .with_signing_key(self.signing_key.as_ref()) } } diff --git a/seaweed-worker/crates/lance/src/jobs/compact.rs b/seaweed-worker/crates/lance/src/jobs/compact.rs index 728437420d..a4ae5d7565 100644 --- a/seaweed-worker/crates/lance/src/jobs/compact.rs +++ b/seaweed-worker/crates/lance/src/jobs/compact.rs @@ -12,7 +12,7 @@ use seaweed_worker_core::pb::{ use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler}; use tracing::warn; -use crate::catalog::{NamespaceClient, parse_id}; +use crate::catalog::{NamespaceClient, SigningKey, parse_id}; use crate::dataset; use crate::jobs::{FORMAT, clamp, observation, string_list, table_id}; @@ -35,6 +35,7 @@ const MIN_FRAGMENTS_CEILING: i64 = 4096; pub struct CompactHandler { namespace_url: String, fallback: dataset::FallbackOptions, + signing_key: Option, metrics: Option, } @@ -43,6 +44,7 @@ impl CompactHandler { Self { namespace_url, fallback: dataset::FallbackOptions::new(), + signing_key: None, metrics: None, } } @@ -58,8 +60,15 @@ impl CompactHandler { self } + pub fn with_signing_key(mut self, signing_key: Option) -> Self { + self.signing_key = signing_key; + self + } + fn client(&self) -> NamespaceClient { - NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback) + NamespaceClient::new(self.namespace_url.clone()) + .with_fallback(&self.fallback) + .with_signing_key(self.signing_key.as_ref()) } } diff --git a/seaweed-worker/crates/lance/src/jobs/indices.rs b/seaweed-worker/crates/lance/src/jobs/indices.rs index e619ff43d5..24f87b0c68 100644 --- a/seaweed-worker/crates/lance/src/jobs/indices.rs +++ b/seaweed-worker/crates/lance/src/jobs/indices.rs @@ -13,7 +13,7 @@ use seaweed_worker_core::pb::{ use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler}; use tracing::warn; -use crate::catalog::{NamespaceClient, parse_id}; +use crate::catalog::{NamespaceClient, SigningKey, parse_id}; use crate::dataset::{self, OpenTable}; use crate::jobs::{clamp, string_list, table_id}; @@ -35,6 +35,7 @@ const MAX_UNINDEXED_CEILING: i64 = 100_000_000; pub struct OptimizeIndicesHandler { namespace_url: String, fallback: dataset::FallbackOptions, + signing_key: Option, metrics: Option, } @@ -43,6 +44,7 @@ impl OptimizeIndicesHandler { Self { namespace_url, fallback: dataset::FallbackOptions::new(), + signing_key: None, metrics: None, } } @@ -57,8 +59,15 @@ impl OptimizeIndicesHandler { self } + pub fn with_signing_key(mut self, signing_key: Option) -> Self { + self.signing_key = signing_key; + self + } + fn client(&self) -> NamespaceClient { - NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback) + NamespaceClient::new(self.namespace_url.clone()) + .with_fallback(&self.fallback) + .with_signing_key(self.signing_key.as_ref()) } } diff --git a/seaweed-worker/crates/lance/src/jobs/mod.rs b/seaweed-worker/crates/lance/src/jobs/mod.rs index b5502a7747..a9c3bcfb51 100644 --- a/seaweed-worker/crates/lance/src/jobs/mod.rs +++ b/seaweed-worker/crates/lance/src/jobs/mod.rs @@ -42,27 +42,32 @@ pub(crate) fn table_id(parameters: &HashMap) -> Option, metrics: Option, ) -> Vec> { vec![ Arc::new( compact::CompactHandler::new(namespace_url.clone()) .with_fallback(fallback.clone()) + .with_signing_key(signing_key.clone()) .with_metrics(metrics.clone()), ), Arc::new( indices::OptimizeIndicesHandler::new(namespace_url.clone()) .with_fallback(fallback.clone()) + .with_signing_key(signing_key.clone()) .with_metrics(metrics.clone()), ), Arc::new( sort::SortHandler::new(namespace_url.clone()) .with_fallback(fallback.clone()) + .with_signing_key(signing_key.clone()) .with_metrics(metrics.clone()), ), Arc::new( cleanup::CleanupVersionsHandler::new(namespace_url) .with_fallback(fallback) + .with_signing_key(signing_key) .with_metrics(metrics), ), ] diff --git a/seaweed-worker/crates/lance/src/jobs/sort.rs b/seaweed-worker/crates/lance/src/jobs/sort.rs index ed8aeaa7ba..5ac6859f6a 100644 --- a/seaweed-worker/crates/lance/src/jobs/sort.rs +++ b/seaweed-worker/crates/lance/src/jobs/sort.rs @@ -23,7 +23,7 @@ use seaweed_worker_sort::{ }; use tracing::warn; -use crate::catalog::{NamespaceClient, parse_id}; +use crate::catalog::{NamespaceClient, SigningKey, parse_id}; use crate::dataset; use crate::jobs::{FORMAT, clamp, observation, string_list, table_id}; @@ -54,6 +54,7 @@ const BYTES_PER_MB: u64 = 1024 * 1024; pub struct SortHandler { namespace_url: String, fallback: dataset::FallbackOptions, + signing_key: Option, metrics: Option, } @@ -62,6 +63,7 @@ impl SortHandler { Self { namespace_url, fallback: dataset::FallbackOptions::new(), + signing_key: None, metrics: None, } } @@ -77,8 +79,15 @@ impl SortHandler { self } + pub fn with_signing_key(mut self, signing_key: Option) -> Self { + self.signing_key = signing_key; + self + } + fn client(&self) -> NamespaceClient { - NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback) + NamespaceClient::new(self.namespace_url.clone()) + .with_fallback(&self.fallback) + .with_signing_key(self.signing_key.as_ref()) } } diff --git a/seaweed-worker/crates/lance/src/main.rs b/seaweed-worker/crates/lance/src/main.rs index 8f688cf357..b47a20bf8a 100644 --- a/seaweed-worker/crates/lance/src/main.rs +++ b/seaweed-worker/crates/lance/src/main.rs @@ -63,7 +63,8 @@ struct Args { /// Shared secret for the admin control stream, the `jwt.filer_signing.key` /// value the Go worker reads from its config file. Required by admins that - /// authenticate the worker stream with a signing key. + /// authenticate the worker stream with a signing key. The Lance namespace + /// takes it too: only a holder gets write credentials to vector indexes. #[arg(long, env = "WEED_FILER_SIGNING_KEY")] filer_signing_key: Option, @@ -136,6 +137,7 @@ async fn main() -> Result<()> { let args = Args::parse(); let tls = args.tls()?; + let signing_key = weed_lance_worker::catalog::SigningKey::new(args.filer_signing_key.clone()); let options = WorkerOptions { admin_address: args.admin, worker_id: args.id, @@ -164,9 +166,10 @@ async fn main() -> Result<()> { } let mut registry = Registry::new().with_preview(Arc::new( - weed_lance_worker::preview::LancePreview::new(args.namespace.clone(), fallback.clone()), + weed_lance_worker::preview::LancePreview::new(args.namespace.clone(), fallback.clone()) + .with_signing_key(signing_key.clone()), )); - for handler in handlers(args.namespace, fallback, Some(lance_metrics)) { + for handler in handlers(args.namespace, fallback, signing_key, Some(lance_metrics)) { registry = registry.register(handler); } diff --git a/seaweed-worker/crates/lance/src/preview.rs b/seaweed-worker/crates/lance/src/preview.rs index 596d8700a0..49dd6e2775 100644 --- a/seaweed-worker/crates/lance/src/preview.rs +++ b/seaweed-worker/crates/lance/src/preview.rs @@ -10,13 +10,14 @@ use async_trait::async_trait; use futures::TryStreamExt; use seaweed_worker_core::{Preview, PreviewProvider}; -use crate::catalog::NamespaceClient; +use crate::catalog::{NamespaceClient, SigningKey}; use crate::dataset::{self, FallbackOptions}; use crate::jobs::FORMAT; pub struct LancePreview { namespace_url: String, fallback: FallbackOptions, + signing_key: Option, } impl LancePreview { @@ -24,8 +25,14 @@ impl LancePreview { Self { namespace_url, fallback, + signing_key: None, } } + + pub fn with_signing_key(mut self, signing_key: Option) -> Self { + self.signing_key = signing_key; + self + } } #[async_trait] @@ -35,7 +42,9 @@ impl PreviewProvider for LancePreview { } async fn preview(&self, object_id: &[String], row_limit: usize) -> Result { - let client = NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback); + let client = NamespaceClient::new(self.namespace_url.clone()) + .with_fallback(&self.fallback) + .with_signing_key(self.signing_key.as_ref()); let table = dataset::open(&client, object_id, &self.fallback).await?; let total_rows = table.dataset.count_rows(None).await.unwrap_or(0) as i64; diff --git a/test/s3vectors/Makefile b/test/s3vectors/Makefile index b5d301243a..5a390bb950 100644 --- a/test/s3vectors/Makefile +++ b/test/s3vectors/Makefile @@ -7,6 +7,7 @@ endif VOLUME := ./weed-volume CARGO_PROFILE ?= release CARGO_TARGET_DIR ?= $(abspath ../../seaweed-vector/target) +override CARGO_TARGET_DIR := $(abspath $(CARGO_TARGET_DIR)) CARGO_OUTPUT := $(CARGO_TARGET_DIR)/$(if $(filter dev,$(CARGO_PROFILE)),debug,$(CARGO_PROFILE)) export CARGO_TARGET_DIR @@ -18,7 +19,7 @@ $(WEED): FORCE go build -o $(WEED) ../../weed $(VEC): FORCE - cargo build --locked --profile $(CARGO_PROFILE) --manifest-path ../../seaweed-vector/Cargo.toml -p weed-vector-server + cd ../../seaweed-vector && cargo build --locked --profile $(CARGO_PROFILE) -p weed-vector-server cp $(CARGO_OUTPUT)/weed-vector $(VEC) ifneq ($(WORKER),) diff --git a/test/s3vectors/conftest.py b/test/s3vectors/conftest.py index dfb5ab116d..9c99a758e7 100644 --- a/test/s3vectors/conftest.py +++ b/test/s3vectors/conftest.py @@ -70,8 +70,18 @@ def stop(process): @pytest.fixture(scope="module", params=combinations(), ids=lambda p: "-".join(p)) def cluster(request, tmp_path_factory): - volume, mode = request.param - directory = tmp_path_factory.mktemp(f"vectors-{volume}-{mode}") + yield from run_cluster(request.param, tmp_path_factory, identities=True) + + +@pytest.fixture(scope="module", params=combinations(), ids=lambda p: "-".join(p)) +def open_cluster(request, tmp_path_factory): + """A cluster with no S3 identities, and an engine given no credentials.""" + yield from run_cluster(request.param, tmp_path_factory, identities=False) + + +def run_cluster(param, tmp_path_factory, identities): + volume, mode = param + directory = tmp_path_factory.mktemp(f"vectors-{volume}-{mode}" + ("" if identities else "-open")) print(f"\nVector integration logs: {directory}") weed = binary("WEED_BINARY", "weed") engine = binary("VECTOR_BINARY", "weed-vector") @@ -89,10 +99,14 @@ def cluster(request, tmp_path_factory): master_addr = f"127.0.0.1:{master}.{master_grpc}" filer_addr = f"127.0.0.1:{filer}.{filer_grpc}" - def start(name, args): + def start(name, args, unset=()): log = stack.enter_context((directory / f"{name}.log").open("w")) env = dict(os.environ, AWS_ACCESS_KEY_ID="test", AWS_SECRET_ACCESS_KEY="test", - AWS_DEFAULT_REGION="us-east-1", AWS_EC2_METADATA_DISABLED="true") + AWS_DEFAULT_REGION="us-east-1", AWS_EC2_METADATA_DISABLED="true", + WEED_S3_SSE_KEY="s3vectors-test") + env.pop("WEED_S3_SSE_KEK", None) + for key in unset: + env.pop(key) process = subprocess.Popen(args, cwd=directory, stdout=log, stderr=log, env=env) processes[name] = process stack.callback(stop, process) @@ -117,14 +131,20 @@ def cluster(request, tmp_path_factory): f"-defaultStoreDir={directory / 'filer'}"]) wait_for(lambda: listening(filer_grpc), processes, "filer not ready") endpoint = f"http://127.0.0.1:{s3}" - start("s3", [weed, "-v=1", "s3", f"-filer={filer_addr}", f"-config={HERE / 's3.json'}", + # weed s3 turns AWS_ACCESS_KEY_ID into an admin identity, and the engine + # would find it as a credential. + if identities: + config, unset = [f"-config={HERE / 's3.json'}"], () + credentials = ["--storage-option", "aws_access_key_id=test", + "--storage-option", "aws_secret_access_key=test"] + else: + config, unset, credentials = [], ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"), [] + start("s3", [weed, "-v=1", "s3", f"-filer={filer_addr}", *config, "-ip=127.0.0.1", "-ip.bind=127.0.0.1", f"-port={s3}", - f"-port.grpc={s3_grpc}", "-port.iceberg=0", f"-port.lance={lance}"]) + f"-port.grpc={s3_grpc}", "-port.iceberg=0", f"-port.lance={lance}"], unset) wait_for(lambda: http_ready(f"http://127.0.0.1:{lance}/v1/namespace/root/list"), processes, "catalog not ready") start("vector", [engine, "--namespace", f"http://127.0.0.1:{lance}", - "--masters", master_addr, "--listen", f"127.0.0.1:{vector}", - "--storage-option", "aws_access_key_id=test", - "--storage-option", "aws_secret_access_key=test"]) + "--masters", master_addr, "--listen", f"127.0.0.1:{vector}", *credentials], unset) wait_for(lambda: listening(vector), processes, "vector engine not ready") wait_for( diff --git a/test/s3vectors/s3vectors_test.py b/test/s3vectors/s3vectors_test.py index 887d7e3191..05f83943ee 100644 --- a/test/s3vectors/s3vectors_test.py +++ b/test/s3vectors/s3vectors_test.py @@ -1,10 +1,12 @@ """Run the S3 Vectors API against Go/Rust volumes and single/distributed queries.""" +import contextlib import json import time import uuid import boto3 +from botocore.config import Config from botocore.exceptions import ClientError import pytest @@ -58,7 +60,7 @@ def test_vector_lifecycle(vectors): assert [h["key"] for h in hits] == ["a", "c"] distances = {h["key"]: h["distance"] for h in hits} assert distances["a"] == pytest.approx(0.0, abs=1e-6) - assert distances["c"] == pytest.approx(0.02, abs=1e-4) + assert distances["c"] == pytest.approx(0.02 ** 0.5, abs=1e-4) assert all(h["metadata"]["color"] == "red" for h in hits) vectors.delete_vectors(vectorBucketName=BUCKET, indexName=INDEX, keys=["b"]) @@ -204,6 +206,32 @@ def test_filter_types(vectors, index): assert sorted(hit["key"] for hit in hits) == ["0", "3"] +def test_filter_arrays(vectors, index): + rows = {"space": ["space", "classic"], "nums": [1, 2.5], "strs": ["1", "2.5"], "flag": [True], "scalar": "space"} + vectors.put_vectors(**index, vectors=[{"key": key, "data": {"float32": [1, 0, 0, 0]}, "metadata": {"tags": tags}} + for key, tags in rows.items()]) + + def matches(condition): + hits = vectors.query_vectors(**index, queryVector={"float32": [1, 0, 0, 0]}, topK=10, + filter={"tags": condition})["vectors"] + return sorted(hit["key"] for hit in hits) + + assert matches("space") == ["scalar", "space"] + assert matches({"$eq": 1}) == ["nums"] + assert matches({"$eq": "1"}) == ["strs"] + assert matches({"$eq": True}) == ["flag"] + assert matches({"$in": ["classic", 2.5]}) == ["nums", "space"] + assert matches({"$ne": "space"}) == ["flag", "nums", "strs"] + assert matches({"$nin": [1, "2.5"]}) == ["flag", "scalar", "space"] + + +def test_euclidean_distance(vectors, index): + vectors.put_vectors(**index, vectors=[{"key": "p", "data": {"float32": [3, 4, 0, 0]}}]) + result = vectors.query_vectors(**index, queryVector={"float32": [0, 0, 0, 0]}, topK=1, returnDistance=True) + assert result["distanceMetric"] == "euclidean" + assert result["vectors"][0]["distance"] == pytest.approx(5.0) + + def test_cosine_metric(vectors): bucket = "cosine-" + uuid.uuid4().hex[:16] vectors.create_vector_bucket(vectorBucketName=bucket) @@ -270,17 +298,44 @@ def test_vector_policy_permissions(vectors, vector_clients, index): vectors.delete_index(**other) +def index_object_encryption(vectors, bucket): + s3 = boto3.client("s3", endpoint_url=vectors.meta.endpoint_url, aws_access_key_id="test", + aws_secret_access_key="test", region_name="us-east-1") + keys = [o["Key"] for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix=f"vector/{INDEX}/") + for o in page.get("Contents", []) if o["Size"] > 0] + assert any("/data/" in key for key in keys) and any("/_versions/" in key for key in keys) + return {key: s3.head_object(Bucket=bucket, Key=key).get("ServerSideEncryption") for key in keys} + + def test_vector_bucket_encryption(vectors): bucket = "encryption-" + uuid.uuid4().hex[:16] + plain = bucket + "-plain" config = {"sseType": "AES256"} vectors.create_vector_bucket(vectorBucketName=bucket, encryptionConfiguration=config) + vectors.create_vector_bucket(vectorBucketName=plain) try: assert vectors.get_vector_bucket(vectorBucketName=bucket)["vectorBucket"]["encryptionConfiguration"] == config expect_error("ValidationException", vectors.create_vector_bucket, vectorBucketName=bucket + "-kms", encryptionConfiguration={"sseType": "aws:kms", "kmsKeyArn": "arn:aws:kms:us-east-1:123456789012:key/test"}) assert vectors.list_vector_buckets(prefix=bucket + "-kms")["vectorBuckets"] == [] + + for name in (bucket, plain): + vectors.create_index(vectorBucketName=name, indexName=INDEX, dataType="float32", dimension=4, + distanceMetric="euclidean") + vectors.put_vectors(vectorBucketName=name, indexName=INDEX, vectors=[ + {"key": "a", "data": {"float32": [1, 0, 0, 0]}, "metadata": {"color": "red"}}, + {"key": "b", "data": {"float32": [0, 1, 0, 0]}, "metadata": {"color": "blue"}}, + ]) + hits = vectors.query_vectors(vectorBucketName=name, indexName=INDEX, queryVector={"float32": [1, 0, 0, 0]}, + topK=1, returnMetadata=True)["vectors"] + assert [(h["key"], h["metadata"]) for h in hits] == [("a", {"color": "red"})] + assert set(index_object_encryption(vectors, bucket).values()) == {"AES256"} + assert set(index_object_encryption(vectors, plain).values()) == {None} finally: - vectors.delete_vector_bucket(vectorBucketName=bucket) + for name in (bucket, plain): + with contextlib.suppress(ClientError): + vectors.delete_index(vectorBucketName=name, indexName=INDEX) + vectors.delete_vector_bucket(vectorBucketName=name) def test_catalog_pagination(vectors, index): @@ -362,6 +417,40 @@ def test_index_arn_validation(vectors, index): assert vectors.get_index(**index)["index"]["indexArn"] == arn +def test_resource_tags(vectors, vector_clients, index): + reader = vector_clients["reader"] + bucket_arn = vectors.get_vector_bucket(vectorBucketName=index["vectorBucketName"])["vectorBucket"]["vectorBucketArn"] + index_arn = vectors.get_index(**index)["index"]["indexArn"] + assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {} + vectors.tag_resource(resourceArn=bucket_arn, tags={"team": "search", "env": "dev"}) + vectors.tag_resource(resourceArn=index_arn, tags={"team": "ranking"}) + assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {"team": "search", "env": "dev"} + assert vectors.list_tags_for_resource(resourceArn=index_arn)["tags"] == {"team": "ranking"} + vectors.untag_resource(resourceArn=bucket_arn, tagKeys=["env", "absent"]) + assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {"team": "search"} + expect_error("ValidationException", vectors.tag_resource, resourceArn=bucket_arn, + tags={f"k{i}": "v" for i in range(50)}) + expect_error("NotFoundException", vectors.list_tags_for_resource, resourceArn=bucket_arn + "/index/missing") + expect_error("AccessDeniedException", reader.list_tags_for_resource, resourceArn=bucket_arn) + expect_error("AccessDeniedException", reader.tag_resource, resourceArn=index_arn, tags={"team": "other"}) + assert vectors.list_tags_for_resource(resourceArn=index_arn)["tags"] == {"team": "ranking"} + + +def test_create_with_tags(vectors): + bucket = "tagged-" + uuid.uuid4().hex[:16] + bucket_arn = vectors.create_vector_bucket(vectorBucketName=bucket, tags={"team": "search"})["vectorBucketArn"] + try: + assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {"team": "search"} + index_arn = vectors.create_index(vectorBucketName=bucket, indexName=INDEX, dataType="float32", dimension=4, + distanceMetric="cosine", tags={"tier": "hot"})["indexArn"] + try: + assert vectors.list_tags_for_resource(resourceArn=index_arn)["tags"] == {"tier": "hot"} + finally: + vectors.delete_index(vectorBucketName=bucket, indexName=INDEX) + finally: + vectors.delete_vector_bucket(vectorBucketName=bucket) + + def test_invalid_credentials(cluster): client = boto3.client("s3vectors", endpoint_url=cluster["endpoint"], aws_access_key_id="unknown", aws_secret_access_key="unknown", region_name="us-east-1") @@ -389,3 +478,18 @@ def test_query_without_writer(vectors, index, cluster): raise time.sleep(0.2) assert [v["key"] for v in result["vectors"]] == ["survivor"] + + +def test_engine_without_credentials(open_cluster): + """With no S3 identities, an engine given no credentials still serves an index.""" + client = boto3.client("s3vectors", endpoint_url=open_cluster["endpoint"], aws_access_key_id="any", + aws_secret_access_key="any", region_name="us-east-1", + config=Config(connect_timeout=3, read_timeout=20, retries={"max_attempts": 0})) + index = {"vectorBucketName": BUCKET, "indexName": INDEX} + client.create_vector_bucket(vectorBucketName=BUCKET) + client.create_index(**index, dataType="float32", dimension=2, distanceMetric="cosine") + client.put_vectors(**index, vectors=[{"key": "a", "data": {"float32": [1, 0]}}, + {"key": "b", "data": {"float32": [0, 1]}}]) + hits = client.query_vectors(**index, queryVector={"float32": [1, 0]}, topK=1)["vectors"] + assert [h["key"] for h in hits] == ["a"] + diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.go index 9c3eb6696f..fd9abe2e0d 100644 --- a/weed/admin/dash/admin_server.go +++ b/weed/admin/dash/admin_server.go @@ -43,6 +43,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/scheduler" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" "github.com/seaweedfs/seaweedfs/weed/worker/tasks" _ "github.com/seaweedfs/seaweedfs/weed/credential/grpc" // Register gRPC credential store @@ -167,14 +168,21 @@ type AdminServer struct { collectionStatsCacheThreshold time.Duration s3TablesManager *s3tables.Manager + s3VectorsServer *s3vectors.Server icebergPort int lancePort int // s3PublicEndpoint is the client-facing S3 address from admin.toml // (s3.public_endpoint); discovered S3 servers are the fallback. s3PublicEndpoint string + + // uploadMaxSize caps the total bytes of one file browser upload; 0 means no limit. + uploadMaxSize int64 } +// DefaultUploadMaxSizeMB is the default total size limit of one file browser upload. +const DefaultUploadMaxSizeMB = 500 + // Type definitions moved to types.go func NewAdminServer(masters string, filerGroup string, templateFS http.FileSystem, dataDir string, icebergPort, lancePort int, s3PublicEndpoint string) *AdminServer { @@ -215,11 +223,14 @@ func NewAdminServer(masters string, filerGroup string, templateFS http.FileSyste icebergPort: icebergPort, lancePort: lancePort, s3PublicEndpoint: normalizeS3PublicEndpoint(s3PublicEndpoint), + uploadMaxSize: DefaultUploadMaxSizeMB << 20, pluginLock: lockManager, adminPresenceLock: presenceLock, bgCancel: bgCancel, } + server.s3VectorsServer = newS3VectorsServer(server) + // Initialize topic retention purger server.topicRetentionPurger = NewTopicRetentionPurger(server) @@ -1570,6 +1581,18 @@ func (s *AdminServer) GetS3Endpoint() string { return s.s3PublicEndpoint } +// SetUploadMaxSize sets the total size limit in bytes of one file browser +// upload; zero or less removes the limit. +func (s *AdminServer) SetUploadMaxSize(maxBytes int64) { + s.uploadMaxSize = max(maxBytes, 0) +} + +// GetUploadMaxSize returns the total size limit in bytes of one file browser +// upload, or 0 when uploads are unlimited. +func (s *AdminServer) GetUploadMaxSize() int64 { + return s.uploadMaxSize +} + // normalizeS3PublicEndpoint trims a trailing slash and drops, with a warning, // a value that is not a plain absolute http or https URL, so the file browser // hides its URL actions instead of copying broken links. Checking the string diff --git a/weed/admin/dash/file_browser_data.go b/weed/admin/dash/file_browser_data.go index 37385f5fb4..7fd14a3e6b 100644 --- a/weed/admin/dash/file_browser_data.go +++ b/weed/admin/dash/file_browser_data.go @@ -53,6 +53,8 @@ type FileBrowserData struct { HasNextPage bool `json:"has_next_page"` LastFileName string `json:"last_file_name"` // Cursor for next page CurrentLastFileName string `json:"current_last_file_name"` // Cursor from current request (for page size changes) + + UploadMaxSize int64 `json:"upload_max_size"` } // GetFileBrowser retrieves file browser data for a given path with cursor-based @@ -240,6 +242,8 @@ func (s *AdminServer) GetFileBrowser(dir string, prefix string, lastFileName str HasNextPage: hasNextPage, LastFileName: lastEntryName, // Store for next page navigation CurrentLastFileName: lastFileName, // Store input cursor for page size changes + + UploadMaxSize: s.uploadMaxSize, }, nil } diff --git a/weed/admin/dash/s3vectors_management.go b/weed/admin/dash/s3vectors_management.go new file mode 100644 index 0000000000..58204fd2bb --- /dev/null +++ b/weed/admin/dash/s3vectors_management.go @@ -0,0 +1,254 @@ +package dash + +import ( + "context" + "errors" + "net/http" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" +) + +type S3VectorsBucketsData struct { + Username string `json:"username"` + Buckets []S3VectorsBucketSummary `json:"buckets"` + TotalBuckets int `json:"total_buckets"` + Truncated bool `json:"truncated,omitempty"` + LastUpdated time.Time `json:"last_updated"` +} + +type S3VectorsBucketSummary struct { + Name string `json:"name"` + ARN string `json:"arn"` + CreatedAt time.Time `json:"createdAt"` + // Encryption is the bucket's sseType, or empty when it declares none. + Encryption string `json:"encryption,omitempty"` +} + +type S3VectorsBucketData struct { + Username string `json:"username"` + Bucket S3VectorsBucketSummary `json:"bucket"` + Policy string `json:"policy,omitempty"` + Indexes []*s3vectors.Index `json:"indexes"` + TotalIndexes int `json:"total_indexes"` + Truncated bool `json:"truncated,omitempty"` + LastUpdated time.Time `json:"last_updated"` +} + +// The S3 Vectors operations reach the filer through the admin server's own +// filer connection. +type s3VectorsFilerClient struct{ s *AdminServer } + +func (c s3VectorsFilerClient) WithFilerClient(_ bool, fn func(client filer_pb.SeaweedFilerClient) error) error { + return c.s.WithFilerClient(fn) +} + +// newS3VectorsServer serves the catalog side of S3 Vectors for the admin +// console. It has no vector engine, so it never runs an operation that needs +// one, such as CreateIndex. +func newS3VectorsServer(s *AdminServer) *s3vectors.Server { + server := s3vectors.NewServer(s3VectorsFilerClient{s}, nil, nil) + server.SetRegion(s3tables.DefaultRegion) + server.SetAccountID(s3tables.DefaultAccountID) + return server +} + +func (s *AdminServer) executeS3VectorsOperation(ctx context.Context, operation string, req, resp interface{}) error { + return s.s3VectorsServer.Execute(ctx, operation, req, resp, s3_constants.AccountAdminId) +} + +const s3VectorsAdminPage = 500 + +func (s *AdminServer) s3VectorsBucketSummary(ctx context.Context, name string) (S3VectorsBucketSummary, error) { + var resp s3vectors.GetVectorBucketResponse + if err := s.executeS3VectorsOperation(ctx, "GetVectorBucket", &s3vectors.GetVectorBucketRequest{VectorBucketName: name}, &resp); err != nil { + return S3VectorsBucketSummary{}, err + } + b := resp.VectorBucket + summary := S3VectorsBucketSummary{Name: b.VectorBucketName, ARN: b.VectorBucketArn, CreatedAt: b.CreationTime} + if sse, ok := b.EncryptionConfiguration["sseType"].(string); ok { + summary.Encryption = sse + } + return summary, nil +} + +func (s *AdminServer) GetS3VectorsBucketsData(ctx context.Context) (S3VectorsBucketsData, error) { + data := S3VectorsBucketsData{Buckets: []S3VectorsBucketSummary{}} + page := s3VectorsAdminPage + req := &s3vectors.ListVectorBucketsRequest{MaxResults: &page} + for { + var resp s3vectors.ListVectorBucketsResponse + if err := s.executeS3VectorsOperation(ctx, "ListVectorBuckets", req, &resp); err != nil { + return S3VectorsBucketsData{}, err + } + for _, b := range resp.VectorBuckets { + if len(data.Buckets) == s3TablesAdminListLimit { + data.Truncated = true + break + } + summary, err := s.s3VectorsBucketSummary(ctx, b.VectorBucketName) + if isS3VectorsNotFound(err) { + continue + } + if err != nil { + return S3VectorsBucketsData{}, err + } + data.Buckets = append(data.Buckets, summary) + } + if data.Truncated || resp.NextToken == "" { + break + } + req.NextToken = resp.NextToken + } + data.TotalBuckets = len(data.Buckets) + data.LastUpdated = time.Now() + return data, nil +} + +func (s *AdminServer) GetS3VectorsBucketData(ctx context.Context, bucket string) (S3VectorsBucketData, error) { + summary, err := s.s3VectorsBucketSummary(ctx, bucket) + if err != nil { + return S3VectorsBucketData{}, err + } + data := S3VectorsBucketData{Bucket: summary, Indexes: []*s3vectors.Index{}, LastUpdated: time.Now()} + var policy s3vectors.GetVectorBucketPolicyResponse + switch err := s.executeS3VectorsOperation(ctx, "GetVectorBucketPolicy", &s3vectors.GetVectorBucketPolicyRequest{VectorBucketName: bucket}, &policy); { + case err == nil: + data.Policy = policy.Policy + case !isS3VectorsNotFound(err): + return S3VectorsBucketData{}, err + } + page := s3VectorsAdminPage + req := &s3vectors.ListIndexesRequest{VectorBucketName: bucket, MaxResults: &page} + for { + var resp s3vectors.ListIndexesResponse + if err := s.executeS3VectorsOperation(ctx, "ListIndexes", req, &resp); err != nil { + return S3VectorsBucketData{}, err + } + for _, summary := range resp.Indexes { + if len(data.Indexes) == s3TablesAdminListLimit { + data.Truncated = true + break + } + var index s3vectors.GetIndexResponse + err := s.executeS3VectorsOperation(ctx, "GetIndex", &s3vectors.GetIndexRequest{VectorBucketName: bucket, IndexName: summary.IndexName}, &index) + if isS3VectorsNotFound(err) { + continue + } + if err != nil { + return S3VectorsBucketData{}, err + } + data.Indexes = append(data.Indexes, index.Index) + } + if data.Truncated || resp.NextToken == "" { + break + } + req.NextToken = resp.NextToken + } + data.TotalIndexes = len(data.Indexes) + return data, nil +} + +func (s *AdminServer) ListS3VectorsBucketsAPI(w http.ResponseWriter, r *http.Request) { + data, err := s.GetS3VectorsBucketsData(r.Context()) + if err != nil { + writeS3VectorsError(w, err) + return + } + writeJSON(w, http.StatusOK, data) +} + +func (s *AdminServer) CreateS3VectorsBucket(w http.ResponseWriter, r *http.Request) { + if !requireSessionCSRFToken(w, r) { + return + } + var req struct { + Name string `json:"name"` + } + if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil { + writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error()) + return + } + var resp s3vectors.CreateVectorBucketResponse + if err := s.executeS3VectorsOperation(r.Context(), "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: req.Name}, &resp); err != nil { + writeS3VectorsError(w, err) + return + } + writeJSON(w, http.StatusCreated, map[string]interface{}{"arn": resp.VectorBucketArn}) +} + +func (s *AdminServer) DeleteS3VectorsBucket(w http.ResponseWriter, r *http.Request) { + if !requireSessionCSRFToken(w, r) { + return + } + bucket := r.URL.Query().Get("bucket") + if bucket == "" { + writeJSONError(w, http.StatusBadRequest, "bucket query parameter is required") + return + } + if err := s.executeS3VectorsOperation(r.Context(), "DeleteVectorBucket", &s3vectors.DeleteVectorBucketRequest{VectorBucketName: bucket}, nil); err != nil { + writeS3VectorsError(w, err) + return + } + writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Vector bucket deleted"}) +} + +func (s *AdminServer) GetS3VectorsBucketAPI(w http.ResponseWriter, r *http.Request) { + bucket := r.URL.Query().Get("bucket") + if bucket == "" { + writeJSONError(w, http.StatusBadRequest, "bucket query parameter is required") + return + } + data, err := s.GetS3VectorsBucketData(r.Context(), bucket) + if err != nil { + writeS3VectorsError(w, err) + return + } + writeJSON(w, http.StatusOK, data) +} + +func (s *AdminServer) DeleteS3VectorsIndex(w http.ResponseWriter, r *http.Request) { + if !requireSessionCSRFToken(w, r) { + return + } + bucket, index := r.URL.Query().Get("bucket"), r.URL.Query().Get("index") + if bucket == "" || index == "" { + writeJSONError(w, http.StatusBadRequest, "bucket and index query parameters are required") + return + } + if err := s.executeS3VectorsOperation(r.Context(), "DeleteIndex", &s3vectors.DeleteIndexRequest{VectorBucketName: bucket, IndexName: index}, nil); err != nil { + writeS3VectorsError(w, err) + return + } + writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Index deleted"}) +} + +func isS3VectorsNotFound(err error) bool { + var e *s3vectors.Error + return errors.As(err, &e) && e.Code == "NotFoundException" +} + +func writeS3VectorsError(w http.ResponseWriter, err error) { + var e *s3vectors.Error + if !errors.As(err, &e) { + writeJSONError(w, http.StatusInternalServerError, err.Error()) + return + } + status := http.StatusInternalServerError + switch e.Code { + case "ValidationException": + status = http.StatusBadRequest + case "AccessDeniedException": + status = http.StatusForbidden + case "NotFoundException": + status = http.StatusNotFound + case "ConflictException": + status = http.StatusConflict + case "ServiceUnavailableException": + status = http.StatusServiceUnavailable + } + writeJSONError(w, status, e.Message) +} diff --git a/weed/admin/dash/s3vectors_management_test.go b/weed/admin/dash/s3vectors_management_test.go new file mode 100644 index 0000000000..bd0c36a65d --- /dev/null +++ b/weed/admin/dash/s3vectors_management_test.go @@ -0,0 +1,174 @@ +package dash + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" +) + +func newS3VectorsTestServer(t *testing.T) (*AdminServer, *s3tables.ManagerClient) { + t.Helper() + fs := s3tablestest.Start(t) + client := s3tables.NewManagerClient(fs.Client) + server := s3vectors.NewServer(client, nil, nil) + server.SetRegion(s3tables.DefaultRegion) + server.SetAccountID(s3tables.DefaultAccountID) + return &AdminServer{s3VectorsServer: server}, client +} + +// createS3VectorsIndexRecord writes the catalog record CreateIndex leaves +// behind; the admin server has no engine to run CreateIndex itself. +func createS3VectorsIndexRecord(t *testing.T, client *s3tables.ManagerClient, bucket, index string, metadata map[string]string) { + t.Helper() + m := s3tables.NewManager() + m.SetTrusted(true) + arn := "arn:aws:s3tables:us-east-1:000000000000:bucket/" + bucket + ctx := context.Background() + err := m.Execute(ctx, client, "CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}}, nil, s3_constants.AccountAdminId) + if err != nil && !strings.Contains(err.Error(), "already exists") { + t.Fatalf("create namespace: %v", err) + } + full, _ := json.Marshal(metadata) + if err := m.Execute(ctx, client, "CreateTable", &s3tables.CreateTableRequest{ + TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}, Name: index, + Format: s3tables.FormatLance, Metadata: &s3tables.TableMetadata{FullMetadata: full}, + }, nil, s3_constants.AccountAdminId); err != nil { + t.Fatalf("create index record: %v", err) + } +} + +func s3VectorsRequest(t *testing.T, handler http.HandlerFunc, method, target, body string) *httptest.ResponseRecorder { + t.Helper() + w := httptest.NewRecorder() + handler(w, httptest.NewRequest(method, target, strings.NewReader(body))) + return w +} + +func TestS3VectorsBucketsListsOnlyVectorBuckets(t *testing.T) { + s, client := newS3VectorsTestServer(t) + ctx := context.Background() + if w := s3VectorsRequest(t, s.CreateS3VectorsBucket, http.MethodPost, "/api/s3vectors/buckets", `{"name":"media"}`); w.Code != http.StatusCreated { + t.Fatalf("create: %d %s", w.Code, w.Body) + } + if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{ + VectorBucketName: "sealed", EncryptionConfiguration: map[string]interface{}{"sseType": "AES256"}, + }, nil); err != nil { + t.Fatalf("create sealed: %v", err) + } + if err := s3tables.NewManager().Execute(ctx, client, "CreateTableBucket", &s3tables.CreateTableBucketRequest{ + Name: "warehouse", Format: s3tables.FormatIceberg, + }, nil, s3_constants.AccountAdminId); err != nil { + t.Fatalf("create iceberg bucket: %v", err) + } + + data, err := s.GetS3VectorsBucketsData(ctx) + if err != nil { + t.Fatal(err) + } + got := map[string]string{} + for _, b := range data.Buckets { + got[b.Name] = b.Encryption + if want := "arn:aws:s3vectors:us-east-1:000000000000:bucket/" + b.Name; b.ARN != want || b.CreatedAt.IsZero() { + t.Errorf("%s: arn %q created %v, want %q and a time", b.Name, b.ARN, b.CreatedAt, want) + } + } + if len(got) != 2 || got["media"] != "" || got["sealed"] != "AES256" || data.TotalBuckets != 2 || data.Truncated { + t.Fatalf("buckets %v (total %d), want media and sealed (AES256) without the iceberg bucket", got, data.TotalBuckets) + } +} + +func TestS3VectorsBucketShowsIndexesAndPolicy(t *testing.T) { + s, client := newS3VectorsTestServer(t) + ctx := context.Background() + if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: "media"}, nil); err != nil { + t.Fatal(err) + } + createS3VectorsIndexRecord(t, client, "media", "movies", map[string]string{ + "dimension": "3", "distanceMetric": "cosine", "nonFilterableMetadataKeys": `["summary"]`, + }) + policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3vectors:QueryVectors","Resource":"*"}]}` + if err := s.executeS3VectorsOperation(ctx, "PutVectorBucketPolicy", &s3vectors.PutVectorBucketPolicyRequest{VectorBucketName: "media", Policy: policy}, nil); err != nil { + t.Fatal(err) + } + + w := s3VectorsRequest(t, s.GetS3VectorsBucketAPI, http.MethodGet, "/api/s3vectors/indexes?bucket=media", "") + if w.Code != http.StatusOK { + t.Fatalf("get: %d %s", w.Code, w.Body) + } + var data S3VectorsBucketData + if err := json.Unmarshal(w.Body.Bytes(), &data); err != nil { + t.Fatal(err) + } + if data.Policy != policy { + t.Errorf("policy %q, want %q", data.Policy, policy) + } + if data.TotalIndexes != 1 { + t.Fatalf("indexes %+v", data.Indexes) + } + idx := data.Indexes[0] + if idx.IndexName != "movies" || idx.Dimension != 3 || idx.DistanceMetric != "cosine" || idx.DataType != "float32" || + idx.MetadataConfiguration == nil || len(idx.MetadataConfiguration.NonFilterableMetadataKeys) != 1 { + t.Fatalf("index %+v", idx) + } + + if w := s3VectorsRequest(t, s.GetS3VectorsBucketAPI, http.MethodGet, "/api/s3vectors/indexes?bucket=missing", ""); w.Code != http.StatusNotFound { + t.Fatalf("missing bucket: %d, want 404", w.Code) + } +} + +func TestS3VectorsDeleteIndexThenBucket(t *testing.T) { + s, client := newS3VectorsTestServer(t) + ctx := context.Background() + if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: "media"}, nil); err != nil { + t.Fatal(err) + } + createS3VectorsIndexRecord(t, client, "media", "movies", map[string]string{"dimension": "3", "distanceMetric": "cosine"}) + + if w := s3VectorsRequest(t, s.DeleteS3VectorsBucket, http.MethodDelete, "/api/s3vectors/buckets?bucket=media", ""); w.Code != http.StatusConflict { + t.Fatalf("delete a bucket with an index: %d %s, want 409", w.Code, w.Body) + } + if w := s3VectorsRequest(t, s.DeleteS3VectorsIndex, http.MethodDelete, "/api/s3vectors/indexes?bucket=media&index=movies", ""); w.Code != http.StatusOK { + t.Fatalf("delete index: %d %s", w.Code, w.Body) + } + // The emptied index namespace does not keep the bucket alive. + if w := s3VectorsRequest(t, s.DeleteS3VectorsBucket, http.MethodDelete, "/api/s3vectors/buckets?bucket=media", ""); w.Code != http.StatusOK { + t.Fatalf("delete bucket: %d %s", w.Code, w.Body) + } + data, err := s.GetS3VectorsBucketsData(ctx) + if err != nil || data.TotalBuckets != 0 { + t.Fatalf("after delete: %+v, %v", data.Buckets, err) + } +} + +func TestS3VectorsBucketsStopAtTheListLimit(t *testing.T) { + s, client := newS3VectorsTestServer(t) + ctx := context.Background() + // The ICEBERG bucket leaves the first page short, so the last page + // would carry the list past the limit. + if err := s3tables.NewManager().Execute(ctx, client, "CreateTableBucket", &s3tables.CreateTableBucketRequest{ + Name: "iceberg", Format: s3tables.FormatIceberg, + }, nil, s3_constants.AccountAdminId); err != nil { + t.Fatal(err) + } + for i := 0; i <= s3TablesAdminListLimit; i++ { + if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: fmt.Sprintf("media-%04d", i)}, nil); err != nil { + t.Fatal(err) + } + } + data, err := s.GetS3VectorsBucketsData(ctx) + if err != nil { + t.Fatal(err) + } + if data.TotalBuckets != s3TablesAdminListLimit || !data.Truncated { + t.Fatalf("listed %d buckets (truncated %v), want the limit %d and truncated", data.TotalBuckets, data.Truncated, s3TablesAdminListLimit) + } +} diff --git a/weed/admin/handlers/admin_handlers.go b/weed/admin/handlers/admin_handlers.go index 007d1e14b1..80018e1d76 100644 --- a/weed/admin/handlers/admin_handlers.go +++ b/weed/admin/handlers/admin_handlers.go @@ -1,6 +1,7 @@ package handlers import ( + "errors" "net/http" "net/url" "strconv" @@ -14,6 +15,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/admin/view/layout" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" "github.com/seaweedfs/seaweedfs/weed/stats" ) @@ -111,6 +113,8 @@ func (h *AdminHandlers) registerUIRoutes(r *mux.Router) { r.HandleFunc("/object-store/groups", h.groupHandlers.ShowGroups).Methods(http.MethodGet) r.HandleFunc("/object-store/service-accounts", h.serviceAccountHandlers.ShowServiceAccounts).Methods(http.MethodGet) r.HandleFunc("/object-store/s3tables/buckets", h.ShowS3TablesBuckets).Methods(http.MethodGet) + r.HandleFunc("/object-store/vector-buckets", h.ShowS3VectorsBuckets).Methods(http.MethodGet) + r.HandleFunc("/object-store/vector-buckets/{bucket}", h.ShowS3VectorsBucketDetails).Methods(http.MethodGet) r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces", h.ShowS3TablesNamespaces).Methods(http.MethodGet) r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces/{namespace}/tables", h.ShowS3TablesTables).Methods(http.MethodGet) r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces/{namespace}/tables/{table}", h.ShowS3TablesTableDetails).Methods(http.MethodGet) @@ -251,6 +255,13 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) { s3TablesApi.HandleFunc("/tags", h.adminServer.ListS3TablesTags).Methods(http.MethodGet) s3TablesApi.Handle("/tags", wrapWrite(h.adminServer.UntagS3TablesResource)).Methods(http.MethodDelete) + s3VectorsApi := api.PathPrefix("/s3vectors").Subrouter() + s3VectorsApi.HandleFunc("/buckets", h.adminServer.ListS3VectorsBucketsAPI).Methods(http.MethodGet) + s3VectorsApi.Handle("/buckets", wrapWrite(h.adminServer.CreateS3VectorsBucket)).Methods(http.MethodPost) + s3VectorsApi.Handle("/buckets", wrapWrite(h.adminServer.DeleteS3VectorsBucket)).Methods(http.MethodDelete) + s3VectorsApi.HandleFunc("/indexes", h.adminServer.GetS3VectorsBucketAPI).Methods(http.MethodGet) + s3VectorsApi.Handle("/indexes", wrapWrite(h.adminServer.DeleteS3VectorsIndex)).Methods(http.MethodDelete) + filesApi := api.PathPrefix("/files").Subrouter() filesApi.Handle("/delete", wrapWrite(h.fileBrowserHandlers.DeleteFile)).Methods(http.MethodDelete) filesApi.Handle("/delete-multiple", wrapWrite(h.fileBrowserHandlers.DeleteMultipleFiles)).Methods(http.MethodDelete) @@ -373,6 +384,45 @@ func (h *AdminHandlers) ShowS3TablesBuckets(w http.ResponseWriter, r *http.Reque } } +// ShowS3VectorsBuckets renders the vector buckets page +func (h *AdminHandlers) ShowS3VectorsBuckets(w http.ResponseWriter, r *http.Request) { + username := h.getUsername(r) + data, err := h.adminServer.GetS3VectorsBucketsData(r.Context()) + if err != nil { + writeJSONError(w, http.StatusInternalServerError, "Failed to get vector buckets: "+err.Error()) + return + } + data.Username = username + + w.Header().Set("Content-Type", "text/html") + viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context())) + if err := layout.Layout(viewCtx, app.S3VectorsBuckets(data)).Render(r.Context(), w); err != nil { + writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error()) + } +} + +// ShowS3VectorsBucketDetails renders one vector bucket with its indexes and policy +func (h *AdminHandlers) ShowS3VectorsBucketDetails(w http.ResponseWriter, r *http.Request) { + username := h.getUsername(r) + data, err := h.adminServer.GetS3VectorsBucketData(r.Context(), mux.Vars(r)["bucket"]) + if err != nil { + status := http.StatusInternalServerError + var e *s3vectors.Error + if errors.As(err, &e) && e.Code == "NotFoundException" { + status = http.StatusNotFound + } + writeJSONError(w, status, "Failed to get vector bucket: "+err.Error()) + return + } + data.Username = username + + w.Header().Set("Content-Type", "text/html") + viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context())) + if err := layout.Layout(viewCtx, app.S3VectorsBucketDetails(data)).Render(r.Context(), w); err != nil { + writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error()) + } +} + // ShowS3TablesNamespaces renders namespaces for a table bucket func (h *AdminHandlers) ShowS3TablesNamespaces(w http.ResponseWriter, r *http.Request) { username := h.getUsername(r) diff --git a/weed/admin/handlers/file_browser_handlers.go b/weed/admin/handlers/file_browser_handlers.go index 085679b768..fb5f122b72 100644 --- a/weed/admin/handlers/file_browser_handlers.go +++ b/weed/admin/handlers/file_browser_handlers.go @@ -4,6 +4,7 @@ import ( "bufio" "compress/gzip" "context" + "errors" "fmt" "io" "mime" @@ -247,25 +248,46 @@ func (h *FileBrowserHandlers) CreateFolder(w http.ResponseWriter, r *http.Reques // UploadFile handles file upload requests func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request) { - // Get the current path + maxSize := h.adminServer.GetUploadMaxSize() + if maxSize > 0 { + r.Body = http.MaxBytesReader(w, r.Body, maxSize+uploadFormOverhead) + } + + // Parts beyond 32MB spill to temporary files instead of memory + err := r.ParseMultipartForm(32 << 20) + if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + writeJSONError(w, http.StatusRequestEntityTooLarge, uploadTooLargeMessage(maxSize)) + return + } + writeJSONError(w, http.StatusBadRequest, "Failed to parse multipart form: "+err.Error()) + return + } + // The router hands us a copy of the request, so the server cannot remove the spooled parts itself + defer r.MultipartForm.RemoveAll() + currentPath := r.FormValue("path") if currentPath == "" { currentPath = "/" } - // Parse multipart form - err := r.ParseMultipartForm(1 << 30) // 1GB max memory for large file uploads - if err != nil { - writeJSONError(w, http.StatusBadRequest, "Failed to parse multipart form: "+err.Error()) - return - } - // Get uploaded files (supports multiple files) files := r.MultipartForm.File["files"] if len(files) == 0 { writeJSONError(w, http.StatusBadRequest, "No files uploaded") return } + if maxSize > 0 { + var totalSize int64 + for _, fileHeader := range files { + totalSize += fileHeader.Size + } + if totalSize > maxSize { + writeJSONError(w, http.StatusRequestEntityTooLarge, uploadTooLargeMessage(maxSize)) + return + } + } var uploadResults []map[string]interface{} var failedUploads []string @@ -327,6 +349,15 @@ func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request) } } +// uploadFormOverhead allows for the multipart headers and the path field on +// top of the file bytes an upload may carry. Go's multipart parser accepts at +// most 1000 parts, whose headers take about half of this. +const uploadFormOverhead = 1 << 20 + +func uploadTooLargeMessage(maxSize int64) string { + return fmt.Sprintf("Total upload size exceeds the %d MB limit", maxSize>>20) +} + // uploadFileToFiler uploads a file to the cluster via filer gRPC + volume // HTTP. This works whether or not the filer is running with -disableHttp=true, // since the bytes never traverse the filer's HTTP listener. The multipart diff --git a/weed/admin/handlers/file_browser_handlers_test.go b/weed/admin/handlers/file_browser_handlers_test.go index f02fead273..179be2a6a0 100644 --- a/weed/admin/handlers/file_browser_handlers_test.go +++ b/weed/admin/handlers/file_browser_handlers_test.go @@ -1,8 +1,12 @@ package handlers import ( + "bytes" + "fmt" + "mime/multipart" "net/http" "net/http/httptest" + "os" "slices" "strings" "testing" @@ -143,3 +147,72 @@ func TestListFolders_RejectsPathOutsideBuckets(t *testing.T) { } } } + +func TestUploadFile_RejectsOversizedUpload(t *testing.T) { + adminServer := &dash.AdminServer{} + adminServer.SetUploadMaxSize(1 << 20) + h := &FileBrowserHandlers{adminServer: adminServer} + + // 1.5MB and two 600KB files fit the body allowance but exceed the file total; + // 3MB trips the body cap. + for _, sizes := range [][]int{{3 << 19}, {600 << 10, 600 << 10}, {3 << 20}} { + var body bytes.Buffer + mw := multipart.NewWriter(&body) + for i, size := range sizes { + part, err := mw.CreateFormFile("files", fmt.Sprintf("big%d.bin", i)) + if err != nil { + t.Fatal(err) + } + part.Write(make([]byte, size)) + } + mw.WriteField("path", "/buckets/b") + mw.Close() + + req := httptest.NewRequest(http.MethodPost, "/api/files/upload", &body) + req.Header.Set("Content-Type", mw.FormDataContentType()) + w := httptest.NewRecorder() + + h.UploadFile(w, req) + + if w.Code != http.StatusRequestEntityTooLarge { + t.Errorf("sizes %v: expected 413, got %d (body: %s)", sizes, w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "1 MB limit") { + t.Errorf("sizes %v: expected the limit in the error, got %s", sizes, w.Body.String()) + } + } +} + +func TestUploadFile_RemovesSpooledParts(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv("TMPDIR", tmpDir) + + adminServer := &dash.AdminServer{} + adminServer.SetUploadMaxSize(34 << 20) + h := &FileBrowserHandlers{adminServer: adminServer} + + // The 33MB part spills to disk; together the parts exceed the limit, so the upload stops there. + var body bytes.Buffer + mw := multipart.NewWriter(&body) + for i, size := range []int{33 << 20, 3 << 19} { + part, err := mw.CreateFormFile("files", fmt.Sprintf("part%d.bin", i)) + if err != nil { + t.Fatal(err) + } + part.Write(make([]byte, size)) + } + mw.Close() + + req := httptest.NewRequest(http.MethodPost, "/api/files/upload", &body) + req.Header.Set("Content-Type", mw.FormDataContentType()) + w := httptest.NewRecorder() + + h.UploadFile(w, req.WithContext(req.Context())) + + if w.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("expected 413, got %d (body: %s)", w.Code, w.Body.String()) + } + if left, _ := os.ReadDir(tmpDir); len(left) != 0 { + t.Errorf("spooled parts left behind: %v", left) + } +} diff --git a/weed/admin/static/js/admin.js b/weed/admin/static/js/admin.js index 5de9b8bd64..77681f26fc 100644 --- a/weed/admin/static/js/admin.js +++ b/weed/admin/static/js/admin.js @@ -1180,10 +1180,9 @@ async function submitUploadFile() { const files = Array.from(fileInput.files); const totalSize = files.reduce((sum, file) => sum + file.size, 0); - // Validate total file size (limit to 500MB for admin interface) - const maxSize = 500 * 1024 * 1024; // 500MB total - if (totalSize > maxSize) { - showErrorMessage('Total file size exceeds 500MB limit. Please select fewer or smaller files.'); + const maxSize = Number(document.getElementById('uploadFileForm').dataset.maxSize) || 0; + if (maxSize > 0 && totalSize > maxSize) { + showErrorMessage(`Total file size exceeds the ${formatBytes(maxSize)} limit. Please select fewer or smaller files.`); return; } diff --git a/weed/admin/static_gz/js/admin.js.gz b/weed/admin/static_gz/js/admin.js.gz index a4ebc6fd53..d6a6220718 100644 Binary files a/weed/admin/static_gz/js/admin.js.gz and b/weed/admin/static_gz/js/admin.js.gz differ diff --git a/weed/admin/view/app/file_browser.templ b/weed/admin/view/app/file_browser.templ index 4367d74698..7dc84c9069 100644 --- a/weed/admin/view/app/file_browser.templ +++ b/weed/admin/view/app/file_browser.templ @@ -310,12 +310,15 @@ templ FileBrowser(data dash.FileBrowserData) {
-
+
Choose one or more files to upload to the current directory. You can select multiple files by holding Ctrl (Cmd on Mac) while clicking. + if data.UploadMaxSize > 0 { + Total upload size limit: { formatBytes(data.UploadMaxSize) }. + }
diff --git a/weed/admin/view/app/file_browser_templ.go b/weed/admin/view/app/file_browser_templ.go index 0554245049..f994bdd27a 100644 --- a/weed/admin/view/app/file_browser_templ.go +++ b/weed/admin/view/app/file_browser_templ.go @@ -852,20 +852,56 @@ func FileBrowser(data dash.FileBrowserData) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 93, "\">
Upload Files
Choose one or more files to upload to the current directory. You can select multiple files by holding Ctrl (Cmd on Mac) while clicking.
Upload Files
0%
Preparing upload...
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, "\">
Choose one or more files to upload to the current directory. You can select multiple files by holding Ctrl (Cmd on Mac) while clicking. ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if data.UploadMaxSize > 0 { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 95, "Total upload size limit: ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var41 string + templ_7745c5c3_Var41, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(data.UploadMaxSize)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 320, Col: 67} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var41)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, ".") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 97, "
0%
Preparing upload...
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/admin/view/app/s3tables_buckets.templ b/weed/admin/view/app/s3tables_buckets.templ index 6533e32545..04ee616219 100644 --- a/weed/admin/view/app/s3tables_buckets.templ +++ b/weed/admin/view/app/s3tables_buckets.templ @@ -6,6 +6,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/admin/dash" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" ) templ S3TablesBuckets(data dash.S3TablesBucketsData) { @@ -160,6 +161,9 @@ templ S3TablesBuckets(data dash.S3TablesBucketsData) { { bucket.Name } { formatLabel(bucket.Format) } + if s3vectors.IsVectorBucketFormat(bucket.Format) { + vector bucket + } { bucket.OwnerAccountID } { bucket.ARN } diff --git a/weed/admin/view/app/s3tables_buckets_templ.go b/weed/admin/view/app/s3tables_buckets_templ.go index f469bf8481..1829d8083a 100644 --- a/weed/admin/view/app/s3tables_buckets_templ.go +++ b/weed/admin/view/app/s3tables_buckets_templ.go @@ -15,6 +15,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/admin/dash" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" ) func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { @@ -46,7 +47,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var2 string templ_7745c5c3_Var2, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.SafeURL(fmt.Sprintf("//localhost:%d/v1/config", data.IcebergPort))).ResolveURL(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 24, Col: 119} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 25, Col: 119} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var2) if templ_7745c5c3_Err != nil { @@ -65,7 +66,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var3 string templ_7745c5c3_Var3, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.SafeURL(fmt.Sprintf("//localhost:%d/v1/table", data.LancePort))).ResolveURL(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 31, Col: 114} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 32, Col: 114} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var3) if templ_7745c5c3_Err != nil { @@ -83,7 +84,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var4 string templ_7745c5c3_Var4, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.IcebergPort)).Resolve(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 38, Col: 91} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 39, Col: 91} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var4) if templ_7745c5c3_Err != nil { @@ -96,7 +97,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var5 string templ_7745c5c3_Var5, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.LancePort)).Resolve(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 38, Col: 145} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 39, Col: 145} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var5) if templ_7745c5c3_Err != nil { @@ -119,7 +120,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var6 string templ_7745c5c3_Var6, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.IcebergPort)).Resolve(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 52, Col: 105} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 53, Col: 105} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var6) if templ_7745c5c3_Err != nil { @@ -132,7 +133,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var7 string templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("http://localhost:%d", data.IcebergPort)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 52, Col: 162} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 53, Col: 162} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) if templ_7745c5c3_Err != nil { @@ -151,7 +152,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var8 string templ_7745c5c3_Var8, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.LancePort)).Resolve(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 58, Col: 103} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 59, Col: 103} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var8) if templ_7745c5c3_Err != nil { @@ -164,7 +165,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var9 string templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("http://localhost:%d", data.LancePort)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 58, Col: 158} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 59, Col: 158} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) if templ_7745c5c3_Err != nil { @@ -193,7 +194,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var10 string templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalBuckets)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 81, Col: 47} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 82, Col: 47} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10)) if templ_7745c5c3_Err != nil { @@ -206,7 +207,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var11 string templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04")) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 100, Col: 54} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 101, Col: 54} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11)) if templ_7745c5c3_Err != nil { @@ -220,7 +221,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var12 string templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 120, Col: 47} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 121, Col: 47} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12)) if templ_7745c5c3_Err != nil { @@ -244,7 +245,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var13 string templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 160, Col: 28} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 161, Col: 28} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13)) if templ_7745c5c3_Err != nil { @@ -283,7 +284,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var16 string templ_7745c5c3_Var16, templ_7745c5c3_Err = templruntime.NewAttributeExpression(formatBadgeTitle(bucket.Format)).Resolve(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 162, Col: 99} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 163, Col: 99} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var16) if templ_7745c5c3_Err != nil { @@ -310,220 +311,243 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component { var templ_7745c5c3_Var17 string templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(formatLabel(bucket.Format)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 162, Col: 130} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 163, Col: 130} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var18 string - templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.OwnerAccountID) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 164, Col: 38} + if s3vectors.IsVectorBucketFormat(bucket.Format) { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "vector bucket") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var19 string - templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN) + templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.OwnerAccountID) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 165, Col: 52} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 168, Col: 38} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var20 string - templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(bucketCatalogPath(bucket.Format, bucket.Name)) + templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 166, Col: 82} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 169, Col: 52} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var21 string - templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04")) + templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(bucketCatalogPath(bucket.Format, bucket.Name)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 167, Col: 60} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 170, Col: 82} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var22 string + templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04")) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 171, Col: 60} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if bucket.PolicyStatementCount > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "Not configured") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "Not configured") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } bucketName, parseErr := s3tables.ParseBucketNameFromARN(bucket.ARN) if parseErr == nil { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "\"> ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "\" data-bucket-name=\"") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var29 string + templ_7745c5c3_Var29, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.Name).Resolve(ctx) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 204, Col: 161} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var29) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "\" title=\"Delete\">
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } if len(data.Buckets) == 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "
No table buckets found

Create your first S3 Tables bucket to get started.

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "
No table buckets found

Create your first S3 Tables bucket to get started.

") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.IcebergPort > 0 || data.LancePort > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "
Client Examples
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "
Client Examples
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.IcebergPort > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "
ICEBERGDuckDB
")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "
ICEBERGDuckDB
")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				var templ_7745c5c3_Var29 string
-				templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(`INSTALL iceberg;
+				var templ_7745c5c3_Var30 string
+				templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(`INSTALL iceberg;
 LOAD iceberg;
 
 CREATE SECRET (
@@ -534,18 +558,18 @@ CREATE SECRET (
 
 SELECT * FROM iceberg_scan('s3://my-table-bucket/my-namespace/my-table');`)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 251, Col: 74}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 255, Col: 74}
 				}
-				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29))
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "
ICEBERGPython (PyIceberg)
")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "
ICEBERGPython (PyIceberg)
")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				var templ_7745c5c3_Var30 string
-				templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(`from pyiceberg.catalog import load_catalog
+				var templ_7745c5c3_Var31 string
+				templ_7745c5c3_Var31, templ_7745c5c3_Err = templ.JoinStringErrs(`from pyiceberg.catalog import load_catalog
 
 catalog = load_catalog(
     name="seaweedfs",
@@ -558,13 +582,13 @@ catalog = load_catalog(
 
 namespaces = catalog.list_namespaces()`)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 268, Col: 39}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 272, Col: 39}
 				}
-				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var31))
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "
")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -574,25 +598,25 @@ namespaces = catalog.list_namespaces()`) templ_7745c5c3_ElementWriter := templ_7745c5c3_Buffer templ_7745c5c3_Buffer := templ.GetBuffer() var templ_7745c5c3_ElementScripts []templ.ComponentScript - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "
0)} - templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_ElementWriter, templ_7745c5c3_Var31...) + var templ_7745c5c3_Var32 = []any{templ.KV("mt-4", data.IcebergPort > 0)} + templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_ElementWriter, templ_7745c5c3_Var32...) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var32 string - templ_7745c5c3_Var32, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.CSSClasses(templ_7745c5c3_Var31).String()).Resolve(ctx) + var templ_7745c5c3_Var33 string + templ_7745c5c3_Var33, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.CSSClasses(templ_7745c5c3_Var32).String()).Resolve(ctx) if templ_7745c5c3_Err != nil { return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 1, Col: 0} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var32) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var33) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "\"") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "\"") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -606,30 +630,30 @@ namespaces = catalog.list_namespaces()`) } templ.ReleaseBuffer(templ_7745c5c3_Buffer) } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, ">LANCEPython (lance-namespace)
")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, ">LANCEPython (lance-namespace)
")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				var templ_7745c5c3_Var33 string
-				templ_7745c5c3_Var33, templ_7745c5c3_Err = templ.JoinStringErrs(`from lance_namespace import connect
+				var templ_7745c5c3_Var34 string
+				templ_7745c5c3_Var34, templ_7745c5c3_Err = templ.JoinStringErrs(`from lance_namespace import connect
 
 ns = connect("rest", {"uri": "http://localhost:` + fmt.Sprintf("%d", data.LancePort) + `"})
 
 ns.list_namespaces(id=["my-table-bucket"])
 ns.describe_table(id=["my-table-bucket", "my-namespace", "my-table"])`)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 281, Col: 70}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 285, Col: 70}
 				}
-				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var33))
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var34))
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "
LANCEPython (pylance)
")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "
LANCEPython (pylance)
")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				var templ_7745c5c3_Var34 string
-				templ_7745c5c3_Var34, templ_7745c5c3_Err = templ.JoinStringErrs(`import lance
+				var templ_7745c5c3_Var35 string
+				templ_7745c5c3_Var35, templ_7745c5c3_Err = templ.JoinStringErrs(`import lance
 from lance_namespace import connect
 
 ns = connect("rest", {"uri": "http://localhost:` + fmt.Sprintf("%d", data.LancePort) + `"})
@@ -638,84 +662,84 @@ table = ns.describe_table(id=["my-table-bucket", "my-namespace", "my-table"])
 # The namespace vends both the location and the credentials to read it.
 dataset = lance.dataset(table.location, storage_options=table.storage_options)`)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 294, Col: 79}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 298, Col: 79}
 				}
-				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var34))
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var35))
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "
")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "
Create Table Bucket
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "
Create Table Bucket
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.IcebergPort > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.LancePort > 0 && data.IcebergPort <= 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else if data.LancePort > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 68, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 69, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, "
Lance Served over the Lance Namespace API. pylance, LanceDB, Ray.
A bucket holds one format. Tables of the other are refused.
The S3 identity that owns this table bucket. Non-admin users can only access table buckets they own.
Optional tags in key=value format.
Delete Table Bucket

Are you sure you want to delete the table bucket ?

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 71, "\" data-lance-port=\"") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var37 string + templ_7745c5c3_Var37, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.LancePort)).Resolve(ctx) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 367, Col: 169} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var37) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 72, "\">A bucket holds one format. Tables of the other are refused.
The S3 identity that owns this table bucket. Non-admin users can only access table buckets they own.
Optional tags in key=value format.
Delete Table Bucket

Are you sure you want to delete the table bucket ?

") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -723,7 +747,7 @@ dataset = lance.dataset(table.location, storage_options=table.storage_options)`) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 70, "
Table Bucket Policy
Provide a policy JSON; use Delete Policy to remove the policy.
Resource Tags
Loading...
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 73, "
Table Bucket Policy
Provide a policy JSON; use Delete Policy to remove the policy.
Resource Tags
Loading...
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/admin/view/app/s3vectors_bucket_details.templ b/weed/admin/view/app/s3vectors_bucket_details.templ new file mode 100644 index 0000000000..8a6bf1c21d --- /dev/null +++ b/weed/admin/view/app/s3vectors_bucket_details.templ @@ -0,0 +1,210 @@ +package app + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "github.com/seaweedfs/seaweedfs/weed/admin/dash" +) + +templ S3VectorsBucketDetails(data dash.S3VectorsBucketData) { +
+
+

+ { data.Bucket.Name } +

+ +
+
+
+
+
+
ARN
+
{ data.Bucket.ARN }
+
Created
+
{ data.Bucket.CreatedAt.Format("2006-01-02 15:04:05") }
+
Encryption
+
+ if data.Bucket.Encryption != "" { + { data.Bucket.Encryption } + } else { + None + } +
+
+
+
+
+
+
+ { fmt.Sprintf("%d indexes", data.TotalIndexes) } + if data.Truncated { + { fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalIndexes) } + } +
+
+
+
+ + + + + + + + + + + + + + for _, index := range data.Indexes { + + + + + + + + + + } + if len(data.Indexes) == 0 { + + + + } + +
NameData TypeDimensionDistance MetricNon-filterable Metadata KeysCreatedActions
+ { index.IndexName } +
{ index.IndexArn }
+ 		{ index.DataType }{ fmt.Sprintf("%d", index.Dimension) }{ index.DistanceMetric } + if index.MetadataConfiguration != nil && len(index.MetadataConfiguration.NonFilterableMetadataKeys) > 0 { + { strings.Join(index.MetadataConfiguration.NonFilterableMetadataKeys, ", ") } + } else { + None + } + { index.CreationTime.Format("2006-01-02 15:04") } + +
+ No indexes. Create one with an S3 Vectors client, e.g. aws s3vectors create-index. +
+
+
+
+
+
+
+ Bucket Policy +
+
+
+ if data.Policy != "" { + 
{ indentJSON(data.Policy) }
+ } else { + No policy. Set one with PutVectorBucketPolicy. + } +
+
+ +
+ +} + +const maxIndentedJSONDepth = 32 + +func indentJSON(s string) string { + if jsonDepth(s) > maxIndentedJSONDepth { + return s + } + var out bytes.Buffer + if json.Indent(&out, []byte(s), "", " ") != nil { + return s + } + return out.String() +} + +func jsonDepth(s string) int { + depth, maxDepth := 0, 0 + inString, escaped := false, false + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case escaped: + escaped = false + case inString: + if c == '\\' { + escaped = true + } else if c == '"' { + inString = false + } + case c == '"': + inString = true + case c == '{' || c == '[': + depth++ + if depth > maxDepth { + maxDepth = depth + } + case c == '}' || c == ']': + depth-- + } + } + return maxDepth +} diff --git a/weed/admin/view/app/s3vectors_bucket_details_templ.go b/weed/admin/view/app/s3vectors_bucket_details_templ.go new file mode 100644 index 0000000000..c74e739b65 --- /dev/null +++ b/weed/admin/view/app/s3vectors_bucket_details_templ.go @@ -0,0 +1,392 @@ +// Code generated by templ - DO NOT EDIT. + +// templ: version: v0.3.1100 + +package app + +//lint:file-ignore SA4006 This context is only used if a nested component is present. + +import "github.com/a-h/templ" +import templruntime "github.com/a-h/templ/runtime" + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "github.com/seaweedfs/seaweedfs/weed/admin/dash" +) + +func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component { + return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { + templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context + if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { + return templ_7745c5c3_CtxErr + } + templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) + if !templ_7745c5c3_IsBuffer { + defer func() { + templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) + if templ_7745c5c3_Err == nil { + templ_7745c5c3_Err = templ_7745c5c3_BufErr + } + }() + } + ctx = templ.InitializeContext(ctx) + ctx = templ.ClearChildren(ctx) + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var2 string + templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.Name) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 16, Col: 69} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "

ARN
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var5 string + templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.ARN) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 30, Col: 49} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "
Created
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var6 string + templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.CreatedAt.Format("2006-01-02 15:04:05")) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 32, Col: 79} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "
Encryption
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if data.Bucket.Encryption != "" { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var7 string + templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.Encryption) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 36, Col: 61} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "None") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var8 string + templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d indexes", data.TotalIndexes)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 47, Col: 84} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, " ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if data.Truncated { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var9 string + templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalIndexes)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 49, Col: 138} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + for _, index := range data.Indexes { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + if len(data.Indexes) == 0 { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "
NameData TypeDimensionDistance MetricNon-filterable Metadata KeysCreatedActions
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var10 string + templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(index.IndexName) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 71, Col: 26} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var11 string + templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(index.IndexArn) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 72, Col: 55} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var12 string + templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(index.DataType) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 74, Col: 28} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var13 string + templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", index.Dimension)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 75, Col: 48} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var14 string + templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(index.DistanceMetric) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 76, Col: 34} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if index.MetadataConfiguration != nil && len(index.MetadataConfiguration.NonFilterableMetadataKeys) > 0 { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var15 string + templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(strings.Join(index.MetadataConfiguration.NonFilterableMetadataKeys, ", ")) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 79, Col: 91} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "None") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var16 string + templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(index.CreationTime.Format("2006-01-02 15:04")) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 84, Col: 59} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "
No indexes. Create one with an S3 Vectors client, e.g. aws s3vectors create-index.
Bucket Policy
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if data.Policy != "" { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "
")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			var templ_7745c5c3_Var18 string
+			templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(indentJSON(data.Policy))
+			if templ_7745c5c3_Err != nil {
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 112, Col: 102}
+			}
+			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "No policy. Set one with PutVectorBucketPolicy.") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "
Delete Index

Delete the index and all its vectors?

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + return nil + }) +} + +const maxIndentedJSONDepth = 32 + +func indentJSON(s string) string { + if jsonDepth(s) > maxIndentedJSONDepth { + return s + } + var out bytes.Buffer + if json.Indent(&out, []byte(s), "", " ") != nil { + return s + } + return out.String() +} + +func jsonDepth(s string) int { + depth, maxDepth := 0, 0 + inString, escaped := false, false + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case escaped: + escaped = false + case inString: + if c == '\\' { + escaped = true + } else if c == '"' { + inString = false + } + case c == '"': + inString = true + case c == '{' || c == '[': + depth++ + if depth > maxDepth { + maxDepth = depth + } + case c == '}' || c == ']': + depth-- + } + } + return maxDepth +} + +var _ = templruntime.GeneratedTemplate diff --git a/weed/admin/view/app/s3vectors_buckets.templ b/weed/admin/view/app/s3vectors_buckets.templ new file mode 100644 index 0000000000..626212dd01 --- /dev/null +++ b/weed/admin/view/app/s3vectors_buckets.templ @@ -0,0 +1,189 @@ +package app + +import ( + "fmt" + "net/url" + + "github.com/seaweedfs/seaweedfs/weed/admin/dash" +) + +templ S3VectorsBuckets(data dash.S3VectorsBucketsData) { +
+

+ Vector Buckets +

+
+
+ +
+
+
+
+ + Vector buckets serve the Amazon S3 Vectors API on the S3 port. Each one is a LANCE table bucket that keeps its indexes in its vector namespace. + Indexes are created and filled with an S3 Vectors client, which needs a running weed-vector engine. +
+
+
+
+ { fmt.Sprintf("%d vector buckets", data.TotalBuckets) } + if data.Truncated { + { fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalBuckets) } + } +
+
+
+
+ + + + + + + + + + + + for _, bucket := range data.Buckets { + + + + + + + + } + if len(data.Buckets) == 0 { + + + + } + +
NameARNEncryptionCreatedActions
+ { bucket.Name } + { bucket.ARN } + if bucket.Encryption != "" { + { bucket.Encryption } + } else { + None + } + { bucket.CreatedAt.Format("2006-01-02 15:04") } +
+ + + + +
+
+ +
+
No vector buckets found
+
+
+
+
+
+ + + +} diff --git a/weed/admin/view/app/s3vectors_buckets_templ.go b/weed/admin/view/app/s3vectors_buckets_templ.go new file mode 100644 index 0000000000..8b486b043a --- /dev/null +++ b/weed/admin/view/app/s3vectors_buckets_templ.go @@ -0,0 +1,202 @@ +// Code generated by templ - DO NOT EDIT. + +// templ: version: v0.3.1100 + +package app + +//lint:file-ignore SA4006 This context is only used if a nested component is present. + +import "github.com/a-h/templ" +import templruntime "github.com/a-h/templ/runtime" + +import ( + "fmt" + "net/url" + + "github.com/seaweedfs/seaweedfs/weed/admin/dash" +) + +func S3VectorsBuckets(data dash.S3VectorsBucketsData) templ.Component { + return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { + templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context + if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { + return templ_7745c5c3_CtxErr + } + templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) + if !templ_7745c5c3_IsBuffer { + defer func() { + templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) + if templ_7745c5c3_Err == nil { + templ_7745c5c3_Err = templ_7745c5c3_BufErr + } + }() + } + ctx = templ.InitializeContext(ctx) + ctx = templ.ClearChildren(ctx) + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

Vector Buckets

Vector buckets serve the Amazon S3 Vectors API on the S3 port. Each one is a LANCE table bucket that keeps its indexes in its vector namespace. Indexes are created and filled with an S3 Vectors client, which needs a running weed-vector engine.
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var2 string + templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d vector buckets", data.TotalBuckets)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 31, Col: 104} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, " ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if data.Truncated { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var3 string + templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalBuckets)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 33, Col: 138} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + for _, bucket := range data.Buckets { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + if len(data.Buckets) == 0 { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "
NameARNEncryptionCreatedActions
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var5 string + templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 53, Col: 110} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var6 string + templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 55, Col: 49} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if bucket.Encryption != "" { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var7 string + templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Encryption) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 58, Col: 60} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "None") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var8 string + templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04")) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 63, Col: 57} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "
No vector buckets found
Create Vector Bucket
3 to 63 lowercase letters, digits and hyphens, starting and ending with a letter or digit.
The bucket is created without encryption. To encrypt it, create it with an S3 Vectors client and encryptionConfiguration, which the S3 gateway checks against its SSE-S3 key.
Delete Vector Bucket

Delete the vector bucket ? A bucket that still has indexes is not deleted.

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + return nil + }) +} + +var _ = templruntime.GeneratedTemplate diff --git a/weed/admin/view/app/s3vectors_buckets_test.go b/weed/admin/view/app/s3vectors_buckets_test.go new file mode 100644 index 0000000000..bae344eb2d --- /dev/null +++ b/weed/admin/view/app/s3vectors_buckets_test.go @@ -0,0 +1,80 @@ +package app + +import ( + "bytes" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/admin/dash" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors" + "github.com/stretchr/testify/require" +) + +func TestS3VectorsBucketsRendersBuckets(t *testing.T) { + data := dash.S3VectorsBucketsData{TotalBuckets: 2, Buckets: []dash.S3VectorsBucketSummary{ + {Name: "media", ARN: "arn:aws:s3vectors:us-east-1:000000000000:bucket/media", CreatedAt: time.Unix(0, 0)}, + {Name: "sealed", ARN: "arn:aws:s3vectors:us-east-1:000000000000:bucket/sealed", Encryption: "AES256", CreatedAt: time.Unix(0, 0)}, + }} + var out bytes.Buffer + require.NoError(t, S3VectorsBuckets(data).Render(dash.WithURLPrefix(t.Context(), "/admin"), &out)) + page := out.String() + require.Contains(t, page, `href="/admin/object-store/vector-buckets/media"`) + require.Contains(t, page, `data-bucket-name="sealed"`) + require.Contains(t, page, `AES256`) + require.Contains(t, page, "2 vector buckets") + require.NotContains(t, page, "No vector buckets found") + require.NotContains(t, page, "showing the first") + + data.Truncated = true + out.Reset() + require.NoError(t, S3VectorsBuckets(data).Render(t.Context(), &out)) + require.Contains(t, out.String(), "showing the first 2; list the rest with an S3 Vectors client") +} + +func TestS3VectorsBucketDetailsRendersIndexesAndPolicy(t *testing.T) { + data := dash.S3VectorsBucketData{ + Bucket: dash.S3VectorsBucketSummary{Name: "media", ARN: "arn:aws:s3vectors:us-east-1:000000000000:bucket/media"}, + Policy: `{"Version":"2012-10-17","Statement":[]}`, + TotalIndexes: 1, + Indexes: []*s3vectors.Index{{ + IndexName: "movies", IndexArn: "arn:aws:s3vectors:us-east-1:000000000000:bucket/media/index/movies", + DataType: "float32", Dimension: 384, DistanceMetric: "cosine", + MetadataConfiguration: &s3vectors.MetadataConfiguration{NonFilterableMetadataKeys: []string{"summary", "body"}}, + }}, + } + var out bytes.Buffer + require.NoError(t, S3VectorsBucketDetails(data).Render(t.Context(), &out)) + page := out.String() + for _, want := range []string{"movies", ">384<", ">cosine<", ">float32<", "summary, body", `data-index-name="movies"`, "{\n "Version": "2012-10-17",\n "Statement": []\n}"} { + require.Contains(t, page, want) + } + require.False(t, strings.Contains(page, "No indexes."), "an index is listed") + + data.Policy = "not json" + out.Reset() + require.NoError(t, S3VectorsBucketDetails(data).Render(t.Context(), &out)) + require.Contains(t, out.String(), ">not json") + + data.Policy = `{"Version":"2012-10-17","Extra":` + strings.Repeat("[", 5000) + strings.Repeat("]", 5000) + "}" + out.Reset() + require.NoError(t, S3VectorsBucketDetails(data).Render(t.Context(), &out)) + require.Contains(t, out.String(), `"Extra":[[[[`, "a deeply nested policy is shown as stored") +} + +func TestJSONDepthSkipsStrings(t *testing.T) { + require.Equal(t, 3, jsonDepth(`{"a":["[{\"[",{}]}`)) + require.Equal(t, 0, jsonDepth(`"[[["`)) +} + +func TestS3TablesBucketsLinksVectorBuckets(t *testing.T) { + data := dash.S3TablesBucketsData{Buckets: []dash.S3TablesBucketSummary{ + {Name: "vectors", ARN: "arn:aws:s3tables:us-east-1:000000000000:bucket/vectors", Format: "LANCE"}, + {Name: "warehouse", ARN: "arn:aws:s3tables:us-east-1:000000000000:bucket/warehouse", Format: "ICEBERG"}, + }} + var out bytes.Buffer + require.NoError(t, S3TablesBuckets(data).Render(t.Context(), &out)) + page := out.String() + require.Contains(t, page, `href="/object-store/vector-buckets/vectors"`) + require.NotContains(t, page, `href="/object-store/vector-buckets/warehouse"`) +} diff --git a/weed/admin/view/layout/layout.templ b/weed/admin/view/layout/layout.templ index 26f228ba4e..3e10120a89 100644 --- a/weed/admin/view/layout/layout.templ +++ b/weed/admin/view/layout/layout.templ @@ -189,6 +189,11 @@ templ Layout(view ViewContext, content templ.Component) { Table Buckets +
  • + + Vector Buckets + +
  • Users diff --git a/weed/admin/view/layout/layout_templ.go b/weed/admin/view/layout/layout_templ.go index 3da50ff198..8cb49a4651 100644 --- a/weed/admin/view/layout/layout_templ.go +++ b/weed/admin/view/layout/layout_templ.go @@ -534,365 +534,378 @@ func Layout(view ViewContext, content templ.Component) templ.Component { return templ_7745c5c3_Err } var templ_7745c5c3_Var32 string - templ_7745c5c3_Var32, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/users")).ResolveURL(ctx) + templ_7745c5c3_Var32, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/vector-buckets")).ResolveURL(ctx) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 193, Col: 84} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 193, Col: 93} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var32) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "\">Users
  • Vector Buckets
  • Groups
  • Users
  • Service Accounts
  • Groups
  • Policies
    • MANAGEMENT
    MANAGEMENT
    • File Browser
    • ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if isMQPage { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "Message Queue ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "Message Queue ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "Message Queue ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "Message Queue ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } if isMQPage { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "
      • ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "
        • ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if currentPath == "/mq/brokers" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "Brokers") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "Brokers") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "\">Brokers") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "
        • ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if currentPath == "/mq/topics" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "Topics") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "\">Brokers") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "
        • ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if currentPath == "/mq/topics" { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "Topics") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "\">Topics") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "Topics") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "
        ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "
      ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "
    WORKERS
    • ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if isDefaultWorkerPage { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "Brokers
    • Default") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "\">Topics
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "
    WORKERS
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 79, "
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -900,108 +913,95 @@ func Layout(view ViewContext, content templ.Component) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 79, "
    © ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var51 string - templ_7745c5c3_Var51, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year())) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 351, Col: 60} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var51)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 80, " SeaweedFS Admin v") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 80, "
    © ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var52 string - templ_7745c5c3_Var52, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER) + templ_7745c5c3_Var52, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year())) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 351, Col: 102} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 356, Col: 60} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var52)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 81, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 81, " SeaweedFS Admin v") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var53 string + templ_7745c5c3_Var53, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 356, Col: 102} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var53)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 82, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if version.COMMIT != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 82, "(") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 83, "(") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var53 string - templ_7745c5c3_Var53, templ_7745c5c3_Err = templ.JoinStringErrs(version.COMMIT) + var templ_7745c5c3_Var54 string + templ_7745c5c3_Var54, templ_7745c5c3_Err = templ.JoinStringErrs(version.COMMIT) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 353, Col: 55} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 358, Col: 55} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var53)) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var54)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 83, ") ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 84, ") ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } if !strings.Contains(version.VERSION, "enterprise") { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 84, "• Enterprise Version Available") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 85, "• Enterprise Version Available") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 85, "
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 91, "\">") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -1059,134 +1072,134 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon ctx = templ.InitializeContext(ctx) ctx = templ.ClearChildren(ctx) prefix := dash.URLPrefixFromContext(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 92, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var61 string - templ_7745c5c3_Var61, templ_7745c5c3_Err = templ.JoinStringErrs(title) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 386, Col: 17} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var61)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 93, " - Login") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var62 string - templ_7745c5c3_Var62, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/favicon.ico").ResolveURL(ctx) + templ_7745c5c3_Var62, templ_7745c5c3_Err = templ.JoinStringErrs(title) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 387, Col: 58} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 391, Col: 17} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var62) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var62)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, "\" type=\"image/x-icon\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><link href=\"") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, " - Login

    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, "\" rel=\"stylesheet\">

    Please sign in to continue

    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 97, "\" rel=\"stylesheet\">

    ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var66 string + templ_7745c5c3_Var66, templ_7745c5c3_Err = templ.JoinStringErrs(title) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 405, Col: 57} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var66)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 98, "

    Please sign in to continue

    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if errorMessage != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 98, "
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 99, "
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var66 string - templ_7745c5c3_Var66, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage) + var templ_7745c5c3_Var67 string + templ_7745c5c3_Var67, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 407, Col: 45} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 412, Col: 45} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var66)) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var67)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 99, "
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 100, "
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 100, "
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 103, "\">
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/command/admin.go b/weed/command/admin.go index a4844f1b8f..59ac89bdb0 100644 --- a/weed/command/admin.go +++ b/weed/command/admin.go @@ -64,6 +64,7 @@ type AdminOptions struct { debugPort *int cpuProfile *string memProfile *string + uploadMaxSizeMB *int // workerGrpcListener, when set, is a listener already bound to grpcPort by // the caller. `weed mini` reserves the port this way because the admin @@ -125,6 +126,7 @@ func init() { a.debugPort = cmdAdmin.Flag.Int("debug.port", 6060, "http port for debugging") a.cpuProfile = cmdAdmin.Flag.String("cpuprofile", "", "cpu profile output file") a.memProfile = cmdAdmin.Flag.String("memprofile", "", "memory profile output file") + a.uploadMaxSizeMB = cmdAdmin.Flag.Int("uploadMaxSizeMB", dash.DefaultUploadMaxSizeMB, "maximum total size in MB of one file browser upload (0 for no limit)") } var cmdAdmin = &Command{ @@ -500,6 +502,9 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool, s3PublicEndpoint = options.defaultS3PublicEndpoint } adminServer := dash.NewAdminServer(*options.master, *options.filerGroup, nil, dataDir, icebergPort, lancePort, s3PublicEndpoint) + if options.uploadMaxSizeMB != nil { + adminServer.SetUploadMaxSize(int64(*options.uploadMaxSizeMB) << 20) + } if err := adminServer.ApplyPluginConfigFromToml(util.GetViper()); err != nil { return fmt.Errorf("apply admin.toml to plugin config: %w", err) diff --git a/weed/command/filer_sync_process_test.go b/weed/command/filer_sync_process_test.go index 132902134a..3e81e0d914 100644 --- a/weed/command/filer_sync_process_test.go +++ b/weed/command/filer_sync_process_test.go @@ -1,11 +1,14 @@ package command import ( + "os" + "path/filepath" "strings" "testing" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/replication/sink" + "github.com/seaweedfs/seaweedfs/weed/replication/sink/localsink" "github.com/seaweedfs/seaweedfs/weed/replication/source" "github.com/seaweedfs/seaweedfs/weed/util" ) @@ -343,3 +346,35 @@ func TestGenProcessFunctionRenameFromExcludedDirBecomesCreate(t *testing.T) { t.Fatalf("unexpected delete/update calls: deletes=%v updates=%v", dataSink.deleteKeys, dataSink.updateKeys) } } + +func TestGenProcessFunctionLocalSinkStaysInsideTarget(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "backup") + dataSink := &localsink.LocalSink{Dir: target} + dataSink.SetSourceFiler(&source.FilerSource{}) + processFn := genProcessFunction("/", target, nil, nil, nil, nil, dataSink, true, false) + + for _, dir := range []string{"/safe", "/../escape"} { + err := processFn(&filer_pb.SubscribeMetadataResponse{ + Directory: dir, + EventNotification: &filer_pb.EventNotification{ + NewParentPath: dir, + NewEntry: &filer_pb.Entry{ + Name: "marker.txt", + Content: []byte("marker"), + Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6}, + }, + }, + }) + if err != nil { + t.Fatalf("processFn %s: %v", dir, err) + } + } + + if _, err := os.Stat(filepath.Join(target, "safe", "marker.txt")); err != nil { + t.Fatalf("inside file missing: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "escape")); !os.IsNotExist(err) { + t.Fatalf("local sink wrote outside %s: %v", target, err) + } +} diff --git a/weed/command/mini.go b/weed/command/mini.go index 498635d302..2d963f7bda 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -17,6 +17,7 @@ import ( "sync" "time" + "github.com/seaweedfs/seaweedfs/weed/admin/dash" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -598,6 +599,7 @@ func initMiniAdminFlags() { miniAdminOptions.allowInsecureBind = cmdMini.Flag.Bool("admin.allowInsecureBind", false, "INSECURE: allow the unauthenticated admin interface to bind to -ip.bind instead of loopback") miniAdminWorkerBindIP = cmdMini.Flag.String("admin.worker.ip", "127.0.0.1", "worker gRPC bind address; expose only with grpc.admin mTLS configured") miniAdminOptions.urlPrefix = cmdMini.Flag.String("admin.urlPrefix", "", "URL path prefix when running the admin UI behind a reverse proxy under a subdirectory (e.g. /seaweedfs)") + miniAdminOptions.uploadMaxSizeMB = cmdMini.Flag.Int("admin.uploadMaxSizeMB", dash.DefaultUploadMaxSizeMB, "maximum total size in MB of one file browser upload (0 for no limit)") } func init() { diff --git a/weed/command/s3.go b/weed/command/s3.go index f74bcda38b..29e9a23f2d 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -675,6 +675,7 @@ func (s3opt *S3Options) startLanceServer(s3ApiServer *s3api.S3ApiServer, listenO lanceServer := lance.NewServer(s3ApiServer, s3ApiServer) lanceServer.SetCredentialValidator(s3ApiServer) + lanceServer.SetSigningKey(s3ApiServer.FilerSigningKey) if s3opt.icebergCredentialRole != nil && *s3opt.icebergCredentialRole != "" { lanceServer.SetCredentialVendor(lanceCredentialVendor{s3ApiServer}) } @@ -723,6 +724,7 @@ func (s3opt *S3Options) vectorsRouter(s3ApiServer *s3api.S3ApiServer, grpcDialOp vectorsServer := s3vectors.NewServer(s3ApiServer, s3ApiServer, backend) vectorsServer.SetRegion(s3tables.DefaultRegion) vectorsServer.SetAccountID(s3tables.DefaultAccountID) + vectorsServer.SetEncryptionChecker(s3api.GetSSES3KeyManager().IsConfigured) vectorsServer.RegisterRoutes(vectorsRouter) return vectorsRouter } diff --git a/weed/filer/filer.go b/weed/filer/filer.go index bce66f5889..bae29d10bf 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -501,6 +501,10 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry, return filer_pb.ErrEntryNameTooLong } + if entry.FullPath.HasDotSegment() { + return fmt.Errorf("invalid path %s", entry.FullPath) + } + if entry.IsDirectory() { entry.Attr.TtlSec = 0 } @@ -794,6 +798,9 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath, } func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry, isFromOtherCluster bool) (err error) { + if entry.FullPath.HasDotSegment() { + return fmt.Errorf("invalid path %s", entry.FullPath) + } ctx = f.syncWriteContext(ctx, entry.FullPath) if oldEntry != nil { entry.Attr.Crtime = oldEntry.Attr.Crtime diff --git a/weed/filer/filer_rename.go b/weed/filer/filer_rename.go index b39d9872e1..7cfd35e88f 100644 --- a/weed/filer/filer_rename.go +++ b/weed/filer/filer_rename.go @@ -10,7 +10,7 @@ import ( func (f *Filer) CanRename(ctx context.Context, source, target util.FullPath, oldName string) error { sourcePath := source.Child(oldName) - if strings.HasPrefix(string(target), string(sourcePath)) { + if util.IsEqualOrUnder(string(target), string(sourcePath)) { return fmt.Errorf("mv: can not move directory to a subdirectory of itself") } diff --git a/weed/filer/filer_rename_test.go b/weed/filer/filer_rename_test.go index d7096963fa..b5447c7539 100644 --- a/weed/filer/filer_rename_test.go +++ b/weed/filer/filer_rename_test.go @@ -1,6 +1,9 @@ package filer import ( + "context" + "os" + "strings" "testing" "github.com/seaweedfs/seaweedfs/weed/util" @@ -29,3 +32,39 @@ func TestDetectBucket(t *testing.T) { }) } } + +func TestCanRenameIntoItself(t *testing.T) { + f, store := newTestFilerWithStubStore(t) + f.DirBucketsPath = "/buckets" + if err := store.InsertEntry(context.Background(), &Entry{FullPath: "/dir/a", Attr: Attr{Mode: os.ModeDir}}); err != nil { + t.Fatal(err) + } + tests := []struct { + name string + oldName string + target string + reject bool + }{ + {"into itself", "a", "/dir/a", true}, + {"into a child", "a", "/dir/a/b", true}, + {"into a deeper descendant", "a", "/dir/a/b/c", true}, + {"into a child with a trailing slash", "a", "/dir/a/b/", true}, + {"old name with a trailing slash into a child", "a/", "/dir/a/b", true}, + {"into a sibling sharing the name prefix", "a", "/dir/a.d", false}, + {"into a descendant of that sibling", "a", "/dir/ab/c", false}, + {"into the parent", "a", "/dir", false}, + {"into an unrelated directory", "a", "/other", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := f.CanRename(context.Background(), "/dir", util.FullPath(tt.target), tt.oldName) + rejected := err != nil && strings.Contains(err.Error(), "subdirectory of itself") + if rejected != tt.reject { + t.Errorf("CanRename(/dir, %q, %q) = %v, want rejected=%v", tt.target, tt.oldName, err, tt.reject) + } + if !tt.reject && err != nil { + t.Errorf("CanRename(/dir, %q, %q) = %v, want nil", tt.target, tt.oldName, err) + } + }) + } +} diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go index 3111e2a5ad..5484383713 100644 --- a/weed/filer/leveldb3/leveldb3_store.go +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -8,6 +8,7 @@ import ( "io" "math" "os" + "path/filepath" "strings" "sync" @@ -167,9 +168,7 @@ func (store *LevelDB3Store) findDB(fullpath weed_util.FullPath, isForChildren bo shortPath = weed_util.FullPath(bucketAndObjectKey[t:]) } - // Dot-prefixed entries directly under /buckets (e.g. .system) are internal - // folders, not S3 buckets; keep them in the default DB by full path. - if strings.HasPrefix(bucket, ".") { + if !isBucketFolder(bucket) { store.dbsLock.RUnlock() return defaultDB, DEFAULT, fullpath, nil } @@ -207,6 +206,14 @@ func (store *LevelDB3Store) emptyBucketDB() (*leveldb.DB, error) { return store.emptyDB, nil } +// Dot-prefixed entries directly under /buckets (e.g. .system) are internal +// folders, not S3 buckets; they stay in the default DB by full path. A bucket +// also names a single folder under store.dir, so it cannot hold a separator; +// "\" is an ordinary name character on Unix and only rejected where it separates. +func isBucketFolder(bucket string) bool { + return !strings.HasPrefix(bucket, ".") && filepath.IsLocal(bucket) && !strings.ContainsAny(bucket, "/"+string(os.PathSeparator)) +} + func (store *LevelDB3Store) createDB(bucket string) (*leveldb.DB, error) { store.dbsLock.Lock() diff --git a/weed/filer/leveldb3/leveldb3_store_bucket.go b/weed/filer/leveldb3/leveldb3_store_bucket.go index feb2a4b049..2b56cad6ca 100644 --- a/weed/filer/leveldb3/leveldb3_store_bucket.go +++ b/weed/filer/leveldb3/leveldb3_store_bucket.go @@ -9,14 +9,18 @@ import ( var _ filer.BucketAware = (*LevelDB3Store)(nil) func (store *LevelDB3Store) OnBucketCreation(bucket string) { + if !isBucketFolder(bucket) { + return + } store.createDB(bucket) } func (store *LevelDB3Store) OnBucketDeletion(bucket string) { - store.closeDB(bucket) - if bucket != "" { // just to make sure - os.RemoveAll(store.dir + "/" + bucket) + if !isBucketFolder(bucket) || bucket == DEFAULT { + return } + store.closeDB(bucket) + os.RemoveAll(store.dir + "/" + bucket) } func (store *LevelDB3Store) CanDropWholeBucket() bool { diff --git a/weed/filer/leveldb3/leveldb3_store_test.go b/weed/filer/leveldb3/leveldb3_store_test.go index a9f3be10da..1f57acd4c1 100644 --- a/weed/filer/leveldb3/leveldb3_store_test.go +++ b/weed/filer/leveldb3/leveldb3_store_test.go @@ -3,6 +3,8 @@ package leveldb import ( "context" "math" + "os" + "path/filepath" "strings" "testing" @@ -207,3 +209,44 @@ func TestListDirectoryPrefixedEntries_StartBelowPrefix(t *testing.T) { t.Fatalf("want [item-b.versions], got %v", names) } } + +func TestBucketEventsStayInsideStoreDir(t *testing.T) { + parent := t.TempDir() + dir := filepath.Join(parent, "store") + store := &LevelDB3Store{} + if err := store.initialize(dir); err != nil { + t.Fatal(err) + } + defer store.Shutdown() + + sentinel := filepath.Join(parent, "keep") + if err := os.WriteFile(sentinel, []byte("keep"), 0644); err != nil { + t.Fatal(err) + } + // Checked after each create, before the matching delete could remove the + // evidence of a db opened in the wrong place. + assertNothingEscaped := func(bucket string) { + t.Helper() + for _, p := range []string{filepath.Join(parent, "outside"), filepath.Join(parent, "CURRENT"), filepath.Join(dir, "x", "y")} { + if _, err := os.Stat(p); !os.IsNotExist(err) { + t.Fatalf("bucket %q event created %s: %v", bucket, p, err) + } + } + } + store.OnBucketCreation("x") + for _, bucket := range []string{"..", ".", "../outside", "", DEFAULT, "x/../" + DEFAULT, "x/y"} { + store.OnBucketCreation(bucket) + assertNothingEscaped(bucket) + store.OnBucketDeletion(bucket) + } + if _, err := os.Stat(sentinel); err != nil { + t.Fatalf("bucket events removed %s: %v", sentinel, err) + } + if _, err := os.Stat(filepath.Join(dir, DEFAULT)); err != nil { + t.Fatalf("default db folder missing: %v", err) + } + ctx := context.Background() + if err := store.InsertEntry(ctx, &filer.Entry{FullPath: "/a/b"}); err != nil { + t.Fatalf("default db unusable after bucket events: %v", err) + } +} diff --git a/weed/operation/chunked_file.go b/weed/operation/chunked_file.go index a5746c8462..5df1647bc1 100644 --- a/weed/operation/chunked_file.go +++ b/weed/operation/chunked_file.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "sort" + "strings" "sync" "google.golang.org/grpc" @@ -94,7 +95,7 @@ func (cm *ChunkManifest) DeleteChunks(masterFn GetMasterFn, usePublicUrl bool, g return nil } -func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (written int64, e error) { +func readChunkNeedle(fileUrl string, w io.Writer, offset int64, size int64, jwt string) (written int64, e error) { req, err := http.NewRequest(http.MethodGet, fileUrl, nil) if err != nil { return written, err @@ -125,7 +126,11 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (wri return written, util_http.StatusError(fileUrl, resp) } - return io.Copy(w, resp.Body) + written, e = io.CopyN(w, resp.Body, size) + if e == io.EOF { + e = fmt.Errorf("%s: %d of %d bytes: %w", fileUrl, written, size, io.ErrUnexpectedEOF) + } + return written, e } func NewChunkedFileReader(chunkList []*ChunkInfo, master pb.ServerAddress, grpcDialOption grpc.DialOption) *ChunkedFileReader { @@ -175,26 +180,69 @@ func (cf *ChunkedFileReader) WriteTo(w io.Writer) (n int64, err error) { return n, ErrInvalidRange } for ; chunkIndex < len(cf.chunkList); chunkIndex++ { - ci := cf.chunkList[chunkIndex] - // if we need read date from local volume server first? - fileUrl, jwt, lookupError := LookupFileId(func(_ context.Context) pb.ServerAddress { - return cf.master - }, cf.grpcDialOption, ci.Fid) - if lookupError != nil { - return n, lookupError - } - if wn, e := readChunkNeedle(fileUrl, w, chunkStartOffset, jwt); e != nil { + wn, e := cf.readChunk(cf.chunkList[chunkIndex], w, chunkStartOffset) + n += wn + cf.pos += wn + if e != nil { return n, e - } else { - n += wn - cf.pos += wn } - chunkStartOffset = 0 } return n, nil } +// readChunk reads the chunk from the first replica that serves it. A failure +// drops the volume's cached locations, and once every cached replica has +// failed the master is asked again. A replica that already sent part of the +// chunk is not continued elsewhere: a volume server streams a needle before +// its checksum is verified, so those bytes may be the corrupt ones. +func (cf *ChunkedFileReader) readChunk(ci *ChunkInfo, w io.Writer, offset int64) (written int64, err error) { + masterFn := func(_ context.Context) pb.ServerAddress { + return cf.master + } + vid, _, _ := strings.Cut(ci.Fid, ",") + dst := &writeErrorRecorder{Writer: w} + tried := make(map[string]bool) + for range 2 { + fileUrls, jwt, lookupError := lookupFileIdUrls(masterFn, cf.grpcDialOption, ci.Fid) + if lookupError != nil { + if err == nil { + err = lookupError + } + return 0, err + } + for _, fileUrl := range fileUrls { + if tried[fileUrl] { + continue + } + tried[fileUrl] = true + written, err = readChunkNeedle(fileUrl, dst, offset, ci.Size-offset, jwt) + if err == nil || dst.err != nil { + return written, err + } + glog.V(0).Infof("read chunk %s from %s: %v", ci.Fid, fileUrl, err) + InvalidateVolumeIdLocationCache(vid) + if written > 0 { + return written, err + } + } + } + return 0, err +} + +type writeErrorRecorder struct { + io.Writer + err error +} + +func (r *writeErrorRecorder) Write(p []byte) (int, error) { + n, err := r.Writer.Write(p) + if err != nil { + r.err = err + } + return n, err +} + func (cf *ChunkedFileReader) ReadAt(p []byte, off int64) (n int, err error) { cf.Seek(off, 0) return cf.Read(p) diff --git a/weed/operation/chunked_file_test.go b/weed/operation/chunked_file_test.go new file mode 100644 index 0000000000..589f77f16d --- /dev/null +++ b/weed/operation/chunked_file_test.go @@ -0,0 +1,257 @@ +package operation + +import ( + "bytes" + "context" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +type fakeVolumeLocationServer struct { + master_pb.UnimplementedSeaweedServer + mu sync.Mutex + calls int + urls []string +} + +func (s *fakeVolumeLocationServer) LookupVolume(_ context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.calls++ + resp := &master_pb.LookupVolumeResponse{} + for _, vid := range req.VolumeOrFileIds { + var locs []*master_pb.Location + for _, u := range s.urls { + locs = append(locs, &master_pb.Location{Url: u}) + } + resp.VolumeIdLocations = append(resp.VolumeIdLocations, &master_pb.LookupVolumeResponse_VolumeIdLocation{ + VolumeOrFileId: vid, + Locations: locs, + }) + } + return resp, nil +} + +func (s *fakeVolumeLocationServer) lookups() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.calls +} + +type chunkReplica struct { + *httptest.Server + mu sync.Mutex + ranges []string +} + +func (c *chunkReplica) host() string { + return strings.TrimPrefix(c.URL, "http://") +} + +func (c *chunkReplica) requests() []string { + c.mu.Lock() + defer c.mu.Unlock() + return append([]string(nil), c.ranges...) +} + +// newChunkReplica serves data; with cutAfter >= 0 it announces the full length +// but drops the connection after cutAfter bytes. +func newChunkReplica(t *testing.T, data []byte, cutAfter int) *chunkReplica { + c := &chunkReplica{} + c.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c.mu.Lock() + c.ranges = append(c.ranges, r.Header.Get("Range")) + c.mu.Unlock() + if cutAfter < 0 { + http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(data)) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + w.WriteHeader(http.StatusOK) + w.Write(data[:cutAfter]) + w.(http.Flusher).Flush() + conn, _, err := w.(http.Hijacker).Hijack() + if err == nil { + conn.Close() + } + })) + t.Cleanup(c.Close) + return c +} + +func deadAddress(t *testing.T) string { + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + addr := lis.Addr().String() + lis.Close() + return addr +} + +func testChunkData(size int) []byte { + data := make([]byte, size) + for i := range data { + data[i] = byte(i * 7) + } + return data +} + +func TestChunkedFileReaderTriesTheNextReplica(t *testing.T) { + const vid = "91201" + data := testChunkData(4096) + healthy := newChunkReplica(t, data, -1) + fake := &fakeVolumeLocationServer{urls: []string{healthy.host()}} + master := startFakeMasterServer(t, fake) + dial := grpc.WithTransportCredentials(insecure.NewCredentials()) + + vc.Set(vid, []Location{{Url: deadAddress(t)}, {Url: healthy.host(), DataInRemote: true}}, time.Minute) + t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) }) + + chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}} + reader := NewChunkedFileReader(chunks, master, dial) + reader.Seek(1000, io.SeekStart) + got, err := io.ReadAll(reader) + reader.Close() + if err != nil { + t.Fatalf("read: %v", err) + } + if !bytes.Equal(got, data[1000:]) { + t.Fatalf("read %d bytes, want the last %d bytes of the chunk", len(got), len(data)-1000) + } + if r := healthy.requests(); len(r) != 1 || r[0] != "bytes=1000-" { + t.Fatalf("healthy replica requests = %q, want one from byte 1000", r) + } + if _, err := vc.Get(vid); err == nil { + t.Fatalf("cached locations kept after a replica failed") + } + if fake.lookups() != 0 { + t.Fatalf("master lookups = %d, want 0", fake.lookups()) + } +} + +func TestChunkedFileReaderDoesNotContinueAPartlySentChunk(t *testing.T) { + const vid = "91205" + data := testChunkData(4096) + broken := newChunkReplica(t, data, 1000) + healthy := newChunkReplica(t, data, -1) + fake := &fakeVolumeLocationServer{urls: []string{healthy.host()}} + master := startFakeMasterServer(t, fake) + dial := grpc.WithTransportCredentials(insecure.NewCredentials()) + + vc.Set(vid, []Location{{Url: broken.host()}, {Url: healthy.host(), DataInRemote: true}}, time.Minute) + t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) }) + + chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}} + reader := NewChunkedFileReader(chunks, master, dial) + _, err := io.ReadAll(reader) + reader.Close() + if err == nil { + t.Fatalf("read of a chunk cut off mid-body succeeded") + } + if r := healthy.requests(); len(r) != 0 { + t.Fatalf("healthy replica requests = %q, want none", r) + } + + reader = NewChunkedFileReader(chunks, master, dial) + got, err := io.ReadAll(reader) + reader.Close() + if err != nil || !bytes.Equal(got, data) { + t.Fatalf("second read: %d bytes, err %v", len(got), err) + } + if fake.lookups() != 1 || len(broken.requests()) != 1 { + t.Fatalf("second read: %d master lookups, %d requests to the broken replica; want 1 and 1", fake.lookups(), len(broken.requests())) + } +} + +func TestChunkedFileReaderAsksMasterWhenCachedReplicasFail(t *testing.T) { + const vid = "91202" + first, second := testChunkData(3000), testChunkData(2000) + healthy := newChunkReplica(t, second, -1) + fake := &fakeVolumeLocationServer{urls: []string{healthy.host()}} + master := startFakeMasterServer(t, fake) + dial := grpc.WithTransportCredentials(insecure.NewCredentials()) + + firstReplica := newChunkReplica(t, first, -1) + const otherVid = "91203" + vc.Set(otherVid, []Location{{Url: firstReplica.host()}}, time.Minute) + vc.Set(vid, []Location{{Url: deadAddress(t)}}, time.Minute) + t.Cleanup(func() { + InvalidateVolumeIdLocationCache(vid) + InvalidateVolumeIdLocationCache(otherVid) + }) + + chunks := []*ChunkInfo{ + {Fid: otherVid + ",01637037d6", Offset: 0, Size: int64(len(first))}, + {Fid: vid + ",02637037d6", Offset: int64(len(first)), Size: int64(len(second))}, + } + reader := NewChunkedFileReader(chunks, master, dial) + defer reader.Close() + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("read: %v", err) + } + if want := append(append([]byte(nil), first...), second...); !bytes.Equal(got, want) { + t.Fatalf("read %d bytes, want %d", len(got), len(want)) + } + if fake.lookups() != 1 { + t.Fatalf("master lookups = %d, want 1", fake.lookups()) + } +} + +func TestChunkedFileReaderTreatsAShortBodyAsFailed(t *testing.T) { + const vid = "91206" + data := testChunkData(2048) + empty := newChunkReplica(t, nil, -1) + healthy := newChunkReplica(t, data, -1) + vc.Set(vid, []Location{{Url: empty.host()}, {Url: healthy.host(), DataInRemote: true}}, time.Minute) + t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) }) + + chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}} + reader := NewChunkedFileReader(chunks, pb.ServerAddress("127.0.0.1:1"), grpc.WithTransportCredentials(insecure.NewCredentials())) + defer reader.Close() + got, err := io.ReadAll(reader) + if err != nil || !bytes.Equal(got, data) { + t.Fatalf("read: %d bytes, err %v; want %d bytes", len(got), err, len(data)) + } + if len(empty.requests()) != 1 || len(healthy.requests()) != 1 { + t.Fatalf("requests: empty %d, healthy %d; want 1 and 1", len(empty.requests()), len(healthy.requests())) + } +} + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("client gone") } + +func TestChunkedFileReaderStopsWhenTheClientGoesAway(t *testing.T) { + const vid = "91204" + data := testChunkData(1024) + a, b := newChunkReplica(t, data, -1), newChunkReplica(t, data, -1) + vc.Set(vid, []Location{{Url: a.host()}, {Url: b.host()}}, time.Minute) + t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) }) + + chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}} + reader := NewChunkedFileReader(chunks, pb.ServerAddress("127.0.0.1:1"), grpc.WithTransportCredentials(insecure.NewCredentials())) + if _, err := reader.WriteTo(failingWriter{}); err == nil || err.Error() != "client gone" { + t.Fatalf("WriteTo error = %v, want the writer's error", err) + } + if n := len(a.requests()) + len(b.requests()); n != 1 { + t.Fatalf("replica requests = %d, want 1", n) + } + if _, err := vc.Get(vid); err != nil { + t.Fatalf("a client write error dropped the cached locations: %v", err) + } +} diff --git a/weed/operation/lookup.go b/weed/operation/lookup.go index a82504277a..77326a10ac 100644 --- a/weed/operation/lookup.go +++ b/weed/operation/lookup.go @@ -51,31 +51,41 @@ var ( // authorization the master stamped on the volume; pass it through to the // volume server on the read request. func LookupFileId(masterFn GetMasterFn, grpcDialOption grpc.DialOption, fileId string) (fullUrl string, jwt string, err error) { + fullUrls, jwt, err := lookupFileIdUrls(masterFn, grpcDialOption, fileId) + if err != nil { + return "", jwt, err + } + return fullUrls[0], jwt, nil +} + +// lookupFileIdUrls returns the read URLs of every replica of fileId, local +// replicas first, each group in random order. +func lookupFileIdUrls(masterFn GetMasterFn, grpcDialOption grpc.DialOption, fileId string) (fullUrls []string, jwt string, err error) { parts := strings.Split(fileId, ",") if len(parts) != 2 { - return "", jwt, errors.New("Invalid fileId " + fileId) + return nil, jwt, errors.New("Invalid fileId " + fileId) } lookup, lookupError := LookupVolumeId(masterFn, grpcDialOption, parts[0]) if lookupError != nil { - return "", jwt, lookupError + return nil, jwt, lookupError } if len(lookup.Locations) == 0 { - return "", jwt, errors.New("File Not Found") + return nil, jwt, errors.New("File Not Found") } - localUrls := make([]string, 0, len(lookup.Locations)) + var localUrls, remoteUrls []string for _, loc := range lookup.Locations { - if !loc.DataInRemote { - localUrls = append(localUrls, loc.Url) - } - } - if len(localUrls) == 0 { - for _, loc := range lookup.Locations { - localUrls = append(localUrls, loc.Url) + fullUrl := "http://" + loc.Url + "/" + fileId + if loc.DataInRemote { + remoteUrls = append(remoteUrls, fullUrl) + } else { + localUrls = append(localUrls, fullUrl) } } + rand.Shuffle(len(localUrls), func(i, j int) { localUrls[i], localUrls[j] = localUrls[j], localUrls[i] }) + rand.Shuffle(len(remoteUrls), func(i, j int) { remoteUrls[i], remoteUrls[j] = remoteUrls[j], remoteUrls[i] }) - return "http://" + localUrls[rand.IntN(len(localUrls))] + "/" + fileId, lookup.Jwt, nil + return append(localUrls, remoteUrls...), lookup.Jwt, nil } func LookupVolumeId(masterFn GetMasterFn, grpcDialOption grpc.DialOption, vid string) (*LookupResult, error) { diff --git a/weed/operation/needle_parse_test.go b/weed/operation/needle_parse_test.go index 339d4507e2..6251ca771d 100644 --- a/weed/operation/needle_parse_test.go +++ b/weed/operation/needle_parse_test.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "io" + "math/rand" "net/http" "testing" @@ -118,6 +119,54 @@ func TestCreateNeedleFromRequest(t *testing.T) { } +func TestUploadKeepsMarginalGzipUncompressed(t *testing.T) { + data := make([]byte, 4300) + rand.New(rand.NewSource(1)).Read(data[:4000]) + gzipped, _ := util.GzipData(data) + if len(gzipped) >= len(data) || len(gzipped)*10 < len(data)*9 { + t.Fatalf("gzip size %d of %d is not a marginal saving", len(gzipped), len(data)) + } + + mockClient := &MockClient{} + var stored *needle.Needle + mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) { + assert.Equal(t, nil, err, "upload: %v", err) + stored = n + } + uploadOption := &UploadOption{ + UploadUrl: "http://localhost:8080/389,0f084d17353afda0", + Filename: "doc.pdf", + MimeType: "application/pdf", + } + newUploader(mockClient).Upload(context.Background(), bytes.NewReader(data), uploadOption) + if stored == nil { + t.Fatal("no needle reached the volume server") + } + assert.False(t, stored.IsCompressed(), "gzip saved less than 10%, data should stay uncompressed") + assert.Equal(t, data, stored.Data) +} + +func TestReplicationKeepsTheSourceMime(t *testing.T) { + for _, mimeType := range []string{"", "application/octet-stream", "Text/Plain; charset=UTF-8"} { + mockClient := &MockClient{} + var stored *needle.Needle + mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) { + assert.Equal(t, nil, err, "upload: %v", err) + stored = n + } + newUploader(mockClient).UploadData(context.Background(), []byte(textContent), &UploadOption{ + UploadUrl: "http://localhost:8080/389,0f084d17353afda0?type=replicate", + Filename: "report.html", + MimeType: mimeType, + IsReplication: true, + }) + if stored == nil { + t.Fatal("no needle reached the volume server") + } + assert.Equal(t, mimeType, string(stored.Mime)) + } +} + var textContent = `Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index a44dc38e8e..71992dfe78 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -390,7 +390,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option option.MimeType = "" } } - if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(option.Filename), option.MimeType); iAmSure && shouldBeCompressed { + if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Ext(option.Filename), option.MimeType); iAmSure && shouldBeCompressed { shouldGzipNow = true } else if !iAmSure && option.MimeType == "" && len(data) > 16*1024 { var compressed []byte @@ -412,7 +412,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option compressed, compressErr := util.GzipData(data) // fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed)) if compressErr == nil { - if len(compressed) < len(data) { + if len(compressed)*10 < len(data)*9 { data = compressed contentIsGzipped = true } @@ -474,6 +474,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option Filename: option.Filename, Cipher: false, IsInputCompressed: contentIsGzipped, + IsReplication: option.IsReplication, MimeType: option.MimeType, PairMap: option.PairMap, Jwt: option.Jwt, @@ -510,7 +511,7 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction // properly handling non-ASCII characters and special characters h.Set("Content-Disposition", mime.FormatMediaType("form-data", map[string]string{"name": "file", "filename": option.Filename})) h.Set("Idempotency-Key", option.UploadUrl) - if option.MimeType == "" { + if option.MimeType == "" && !option.IsReplication { option.MimeType = mime.TypeByExtension(strings.ToLower(filepath.Ext(option.Filename))) } if option.MimeType != "" { diff --git a/weed/replication/sink/localsink/local_sink.go b/weed/replication/sink/localsink/local_sink.go index 8debf7993a..b66e8abff4 100644 --- a/weed/replication/sink/localsink/local_sink.go +++ b/weed/replication/sink/localsink/local_sink.go @@ -3,6 +3,7 @@ package localsink import ( "context" "fmt" + "math/rand/v2" "os" "path/filepath" "strings" @@ -39,6 +40,34 @@ func (localsink *LocalSink) isMultiPartEntry(key string) bool { return strings.HasSuffix(key, ".part") && strings.Contains(key, "/"+s3_constants.MultipartUploadsFolder+"/") } +func (localsink *LocalSink) isInsideDir(key string) (string, bool) { + rel, err := filepath.Rel(filepath.Clean(localsink.Dir), filepath.Clean(key)) + if err == nil && filepath.IsLocal(rel) { + return rel, true + } + glog.Warningf("local sink: skip %s outside %s", key, localsink.Dir) + return "", false +} + +// resolveInsideDir resolves symlinks in the parent of rel while it stays inside the sink directory. +func (localsink *LocalSink) resolveInsideDir(rel string) string { + base, err := filepath.EvalSymlinks(localsink.Dir) + if err != nil { + return rel + } + dir, rest := filepath.Dir(rel), filepath.Base(rel) + for dir != "." { + if resolved, err := filepath.EvalSymlinks(filepath.Join(base, dir)); err == nil { + if inside, err := filepath.Rel(base, resolved); err == nil && filepath.IsLocal(inside) { + return filepath.Join(inside, rest) + } + return rel + } + dir, rest = filepath.Dir(dir), filepath.Join(filepath.Base(dir), rest) + } + return rel +} + func (localsink *LocalSink) initialize(dir string, isIncremental bool) error { localsink.Dir = dir localsink.isIncremental = isIncremental @@ -69,8 +98,17 @@ func (localsink *LocalSink) DeleteEntry(key string, isDirectory, deleteIncludeCh if localsink.isMultiPartEntry(key) { return nil } + rel, ok := localsink.isInsideDir(key) + if !ok { + return nil + } glog.V(4).Infof("Delete Entry key: %s", key) - if err := os.Remove(key); err != nil { + root, err := os.OpenRoot(localsink.Dir) + if err == nil { + err = root.Remove(localsink.resolveInsideDir(rel)) + root.Close() + } + if err != nil { glog.V(0).Infof("remove entry key %s: %s", key, err) } return nil @@ -81,6 +119,10 @@ func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signa if entry.IsDirectory || localsink.isMultiPartEntry(key) { return nil } + rel, ok := localsink.isInsideDir(key) + if !ok { + return nil + } glog.V(4).Infof("Create Entry key: %s", key) totalSize := filer.FileSize(entry) @@ -89,35 +131,37 @@ func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signa return fmt.Errorf("resolve chunks of %s: %w", key, err) } - dir := filepath.Dir(key) - - if _, err := os.Stat(dir); os.IsNotExist(err) { - glog.V(4).Infof("Create Directory key: %s", dir) - if err = os.MkdirAll(dir, 0755); err != nil { - return err - } + if err := os.MkdirAll(localsink.Dir, 0755); err != nil { + return err } + root, err := os.OpenRoot(localsink.Dir) + if err != nil { + return err + } + defer root.Close() - if entry.IsDirectory { - return os.Mkdir(key, os.FileMode(entry.Attributes.FileMode)) + rel = localsink.resolveInsideDir(rel) + dir := filepath.Dir(rel) + if err := root.MkdirAll(dir, 0755); err != nil { + return err } mode := os.FileMode(entry.Attributes.FileMode) - shortFileName := util.ToShortFileName(key) + shortFileName := util.ToShortFileName(rel) // Write to a temp file in the same directory, then atomically rename // on success. This prevents leaving a truncated/empty file if // decryption or chunk copy fails. - tmpFile, err := os.CreateTemp(dir, ".seaweedfs-tmp-*") + tmpName := filepath.Join(dir, fmt.Sprintf(".seaweedfs-tmp-%d", rand.Uint64())) + tmpFile, err := root.OpenFile(tmpName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } - tmpName := tmpFile.Name() defer func() { // Clean up temp file on any error (rename removes it on success) if tmpFile != nil { tmpFile.Close() - os.Remove(tmpName) + root.Remove(tmpName) } }() @@ -151,8 +195,8 @@ func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signa tmpFile = nil // prevent deferred cleanup // Atomic rename into final destination - if err := os.Rename(tmpName, shortFileName); err != nil { - os.Remove(tmpName) + if err := root.Rename(tmpName, shortFileName); err != nil { + root.Remove(tmpName) return err } @@ -164,6 +208,9 @@ func (localsink *LocalSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, ne if localsink.isMultiPartEntry(key) { return true, nil } + if _, ok := localsink.isInsideDir(key); !ok { + return true, nil + } glog.V(4).Infof("Update Entry key: %s", key) // do delete and create foundExistingEntry = util.FileExists(key) diff --git a/weed/replication/sink/localsink/local_sink_dir_test.go b/weed/replication/sink/localsink/local_sink_dir_test.go new file mode 100644 index 0000000000..c044ca500a --- /dev/null +++ b/weed/replication/sink/localsink/local_sink_dir_test.go @@ -0,0 +1,146 @@ +package localsink + +import ( + "os" + "path/filepath" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/replication/source" +) + +func TestLocalSinkStaysInsideDir(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "backup") + sink := &LocalSink{} + sink.initialize(dir, false) + sink.SetSourceFiler(&source.FilerSource{}) + + entry := func() *filer_pb.Entry { + return &filer_pb.Entry{ + Name: "marker.txt", + Content: []byte("marker"), + Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6}, + } + } + + inside := filepath.Join(dir, "safe", "marker.txt") + if err := sink.CreateEntry(inside, entry(), nil); err != nil { + t.Fatalf("create inside: %v", err) + } + if _, err := os.Stat(inside); err != nil { + t.Fatalf("inside file missing: %v", err) + } + + outside := filepath.Join(dir, "..", "escape", "marker.txt") + if err := sink.CreateEntry(outside, entry(), nil); err != nil { + t.Fatalf("create outside: %v", err) + } + if _, err := sink.UpdateEntry(outside, nil, "", entry(), false, nil); err != nil { + t.Fatalf("update outside: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "escape")); !os.IsNotExist(err) { + t.Fatalf("sink wrote outside %s: %v", dir, err) + } + + victim := filepath.Join(root, "victim.txt") + if err := os.WriteFile(victim, []byte("keep"), 0644); err != nil { + t.Fatal(err) + } + if err := sink.DeleteEntry(filepath.Join(dir, "..", "victim.txt"), false, false, nil); err != nil { + t.Fatalf("delete outside: %v", err) + } + if _, err := os.Stat(victim); err != nil { + t.Fatalf("sink deleted a file outside %s: %v", dir, err) + } +} + +func TestLocalSinkDoesNotFollowSymlinksOutOfDir(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "backup") + outside := filepath.Join(root, "outside") + for _, d := range []string{dir, outside} { + if err := os.MkdirAll(d, 0755); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink(outside, filepath.Join(dir, "link")); err != nil { + t.Skipf("symlink: %v", err) + } + sink := &LocalSink{} + sink.initialize(dir, false) + sink.SetSourceFiler(&source.FilerSource{}) + + entry := &filer_pb.Entry{ + Name: "marker.txt", + Content: []byte("marker"), + Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6}, + } + if err := sink.CreateEntry(filepath.Join(dir, "link", "marker.txt"), entry, nil); err == nil { + t.Fatalf("create through a symlink out of %s succeeded", dir) + } + if _, err := os.Stat(filepath.Join(outside, "marker.txt")); !os.IsNotExist(err) { + t.Fatalf("sink wrote through a symlink out of %s: %v", dir, err) + } + + victim := filepath.Join(outside, "victim.txt") + if err := os.WriteFile(victim, []byte("keep"), 0644); err != nil { + t.Fatal(err) + } + if err := sink.DeleteEntry(filepath.Join(dir, "link", "victim.txt"), false, false, nil); err != nil { + t.Fatalf("delete through link: %v", err) + } + if _, err := os.Stat(victim); err != nil { + t.Fatalf("sink deleted through a symlink out of %s: %v", dir, err) + } + + if err := os.Mkdir(filepath.Join(dir, "real"), 0755); err != nil { + t.Fatal(err) + } + if err := os.Symlink("real", filepath.Join(dir, "inner")); err != nil { + t.Fatal(err) + } + if err := sink.CreateEntry(filepath.Join(dir, "inner", "marker.txt"), entry, nil); err != nil { + t.Fatalf("create through a symlink inside %s: %v", dir, err) + } + if _, err := os.Stat(filepath.Join(dir, "real", "marker.txt")); err != nil { + t.Fatalf("inside symlink target missing: %v", err) + } +} + +func TestLocalSinkFollowsAbsoluteSymlinksInsideDir(t *testing.T) { + dir := filepath.Join(t.TempDir(), "backup") + target := filepath.Join(dir, "real") + if err := os.MkdirAll(target, 0755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(dir, "abs")); err != nil { + t.Skipf("symlink: %v", err) + } + sink := &LocalSink{} + sink.initialize(dir, false) + sink.SetSourceFiler(&source.FilerSource{}) + + entry := &filer_pb.Entry{ + Name: "marker.txt", + Content: []byte("marker"), + Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6}, + } + for _, name := range []string{"marker.txt", filepath.Join("sub", "marker.txt")} { + if err := sink.CreateEntry(filepath.Join(dir, "abs", name), entry, nil); err != nil { + t.Fatalf("create %s through an absolute symlink inside %s: %v", name, dir, err) + } + if _, err := os.Stat(filepath.Join(target, name)); err != nil { + t.Fatalf("absolute symlink target %s missing: %v", name, err) + } + if err := sink.DeleteEntry(filepath.Join(dir, "abs", name), false, false, nil); err != nil { + t.Fatalf("delete %s through an absolute symlink: %v", name, err) + } + if _, err := os.Stat(filepath.Join(target, name)); !os.IsNotExist(err) { + t.Fatalf("delete through an absolute symlink left %s: %v", name, err) + } + } + if _, err := os.Lstat(filepath.Join(dir, "abs")); err != nil { + t.Fatalf("symlink itself removed: %v", err) + } +} diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 5f96669554..b86c14b42c 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -25,6 +25,7 @@ import ( "crypto/subtle" "encoding/base64" "encoding/hex" + "errors" "io" "net" "net/http" @@ -74,22 +75,19 @@ func streamHashRequestBody(r *http.Request, sizeLimit int64) (string, error) { return emptySHA256, nil } - limitedReader := io.LimitReader(r.Body, sizeLimit) - hasher := sha256.New() - var bodyBuffer bytes.Buffer - - // Use io.Copy with an io.MultiWriter to hash and buffer the body simultaneously. - if _, err := io.Copy(io.MultiWriter(hasher, &bodyBuffer), limitedReader); err != nil { + body, err := readRequestBody(r, sizeLimit) + if err != nil { return "", err } - r.Body = io.NopCloser(bytes.NewReader(bodyBuffer.Bytes())) + r.Body = io.NopCloser(bytes.NewReader(body)) - if bodyBuffer.Len() == 0 { + if len(body) == 0 { return emptySHA256, nil } - return hex.EncodeToString(hasher.Sum(nil)), nil + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]), nil } // getContentSha256Cksum retrieves the "x-amz-content-sha256" header value. @@ -374,35 +372,29 @@ func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCh if trustedPeer { forwardedPrefix = r.Header.Get("X-Forwarded-Prefix") } + // 9. Candidate paths: the escaped path as sent, then the decoded path that + // an S3 client encodes once when signing; each with the forwarded prefix first. + signaturePaths := []string{pathForSignature} + if decodedPath, decodeErr := url.PathUnescape(pathForSignature); decodeErr == nil && decodedPath != pathForSignature { + signaturePaths = append(signaturePaths, decodedPath) + } + if forwardedPrefix != "" { + prefixedPaths := make([]string, 0, 2*len(signaturePaths)) + for _, p := range signaturePaths { + prefixedPaths = append(prefixedPaths, buildPathWithForwardedPrefix(forwardedPrefix, p)) + } + signaturePaths = append(prefixedPaths, signaturePaths...) + } var matchedHost string +hostCandidates: for i, hostCandidate := range extractHostHeaderCandidates(r, iam.externalHost) { if i > 0 && !replaceSignedHostHeader(extractedSignedHeaders, hostCandidate) { break } - - // 9. Verify with the X-Forwarded-Prefix path first - if forwardedPrefix != "" { - cleanedPath := buildPathWithForwardedPrefix(forwardedPrefix, pathForSignature) - calculatedSignature, errCode = verify(cleanedPath) - if errCode == s3err.ErrNone { + for _, p := range signaturePaths { + if calculatedSignature, errCode = verify(p); errCode == s3err.ErrNone { matchedHost = hostCandidate - break - } - } - - // 10. Verify with the original path - calculatedSignature, errCode = verify(pathForSignature) - if errCode == s3err.ErrNone { - matchedHost = hostCandidate - break - } - - // 11. Retry with decoded path if signature used raw path encoding - if decodedPath, decodeErr := url.PathUnescape(pathForSignature); decodeErr == nil && decodedPath != pathForSignature { - calculatedSignature, errCode = verify(decodedPath) - if errCode == s3err.ErrNone { - matchedHost = hostCandidate - break + break hostCandidates } } } @@ -661,8 +653,12 @@ func extractV4AuthInfoFromHeader(r *http.Request) (*v4AuthInfo, s3err.ErrorCode) hashedPayload := getContentSha256Cksum(r) if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil { + sizeLimit := s3_constants.GetRequestBodyLimitFromContext(r, iamRequestBodyLimit) var hashErr error - hashedPayload, hashErr = streamHashRequestBody(r, iamRequestBodyLimit) + hashedPayload, hashErr = streamHashRequestBody(r, sizeLimit) + if errors.Is(hashErr, errRequestBodyTooLarge) { + return nil, s3err.ErrEntityTooLarge + } if hashErr != nil { return nil, s3err.ErrInternalError } diff --git a/weed/s3api/auth_signature_v4_test.go b/weed/s3api/auth_signature_v4_test.go index a426357b02..e5abf46606 100644 --- a/weed/s3api/auth_signature_v4_test.go +++ b/weed/s3api/auth_signature_v4_test.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "net/http" + "net/url" "slices" "sync" "testing" @@ -735,3 +736,35 @@ func TestVerifyV4SignatureUsesOriginalRequestPath(t *testing.T) { t.Fatalf("signed request rejected after path rewrite: %d", errCode) } } + +// TestSignatureV4WithForwardedPrefixUnicodeKey guards against a signature +// mismatch when an object key contains non-ASCII characters and the request is +// received through a trusted reverse proxy that strips a path prefix. The +// client signs the single percent-encoded path including the prefix, so the +// server must decode the forwarded (already encoded) path before re-encoding it +// together with the prefix. +func TestSignatureV4WithForwardedPrefixUnicodeKey(t *testing.T) { + iam := newTestIAMWithTrustedProxy() + + bucket := "test-bucket" + key := "たあいまd·.jpg" + + // The proxy strips the X-Forwarded-Prefix and forwards the percent-encoded + // object path (the server sees the encoded form, not the raw Unicode key). + r, err := newTestRequest("GET", "https://example.com/"+bucket+"/"+url.PathEscape(key), 0, nil) + if err != nil { + t.Fatalf("failed to create test request: %v", err) + } + + r.Header.Set("X-Forwarded-Prefix", "/s3") + r.Header.Set("Host", "example.com") + r.RemoteAddr = "10.0.0.1:1234" + + // The client signs the full URL path including the prefix, using the single + // percent-encoded form of the Unicode key. + signV4WithPath(r, "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "/s3/"+bucket+"/"+key) + + if _, _, errCode := iam.doesSignatureMatch(r); errCode != s3err.ErrNone { + t.Fatalf("signature verification failed for unicode key behind forwarded prefix: code %d", int(errCode)) + } +} diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go index e7747515d6..3ba5b874e1 100644 --- a/weed/s3api/auto_signature_v4_test.go +++ b/weed/s3api/auto_signature_v4_test.go @@ -16,6 +16,8 @@ import ( "time" "unicode/utf8" + "github.com/aws/aws-sdk-go/aws/credentials" + v4 "github.com/aws/aws-sdk-go/aws/signer/v4" "github.com/gorilla/mux" "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine" @@ -1820,18 +1822,10 @@ func TestIAMLargeBodySecurityLimit(t *testing.T) { ", SignedHeaders=content-type;host;x-amz-date, Signature=dummysignature" req.Header.Set("Authorization", authHeader) - // The function should complete successfully but limit the body to 10 MiB identity, _, errCode := iam.doesSignatureMatch(req) - // Should get signature mismatch (dummy signature) but not internal error - assert.Equal(t, s3err.ErrSignatureDoesNotMatch, errCode) + assert.Equal(t, s3err.ErrEntityTooLarge, errCode) assert.Nil(t, identity) - - // Verify the body was truncated to the limit (10 MiB) - bodyBytes, err := io.ReadAll(req.Body) - assert.NoError(t, err) - assert.Equal(t, 10*(1<<20), len(bodyBytes)) // Should be exactly 10 MiB - assert.Equal(t, strings.Repeat("A", 10*(1<<20)), string(bodyBytes)) // All As, but truncated } // Test the streaming hash implementation directly @@ -1922,30 +1916,65 @@ func TestStreamingVsNonStreamingConsistency(t *testing.T) { } } -// Test streaming with size limit enforcement func TestStreamingWithSizeLimit(t *testing.T) { - // Create a payload larger than the limit - largePayload := strings.Repeat("A", 11*(1<<20)) // 11 MiB - - req, err := http.NewRequest("POST", "http://localhost:8111/", strings.NewReader(largePayload)) + exact := strings.Repeat("A", 1<<20) + req, err := http.NewRequest("POST", "http://localhost:8111/", strings.NewReader(exact)) assert.NoError(t, err) - - // Stream with the limit - hash, err := streamHashRequestBody(req, iamRequestBodyLimit) + hash, err := streamHashRequestBody(req, 1<<20) assert.NoError(t, err) + assert.Equal(t, getSHA256Hash([]byte(exact)), hash) - // Verify the hash is computed for the truncated content (10 MiB) - truncatedPayload := strings.Repeat("A", 10*(1<<20)) - expectedHash := sha256.Sum256([]byte(truncatedPayload)) - expectedHashStr := hex.EncodeToString(expectedHash[:]) - - assert.Equal(t, expectedHashStr, hash) - - // Verify the body was truncated - bodyBytes, err := io.ReadAll(req.Body) + req, err = http.NewRequest("POST", "http://localhost:8111/", strings.NewReader(exact+"A")) assert.NoError(t, err) - assert.Equal(t, 10*(1<<20), len(bodyBytes)) - assert.Equal(t, truncatedPayload, string(bodyBytes)) + _, err = streamHashRequestBody(req, 1<<20) + assert.ErrorIs(t, err, errRequestBodyTooLarge) +} + +func TestNonS3BodyHashLimitFollowsRoute(t *testing.T) { + iam := &IdentityAccessManagement{} + assert.NoError(t, iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{{ + Name: "vec", + Credentials: []*iam_pb.Credential{{AccessKey: "vec_key", SecretKey: "vec_secret"}}, + Actions: []string{"Admin"}, + }}, + })) + signer := v4.NewSigner(credentials.NewStaticCredentials("vec_key", "vec_secret", "")) + + vectorsRoute := int64(s3_constants.S3VectorsMaxRequestBodyBytes) + for _, tc := range []struct { + service string + routeLimit int64 + size int + want s3err.ErrorCode + }{ + {"s3vectors", vectorsRoute, 20 << 20, s3err.ErrNone}, + {"s3vectors", vectorsRoute, s3_constants.S3VectorsMaxRequestBodyBytes, s3err.ErrNone}, + {"s3vectors", vectorsRoute, s3_constants.S3VectorsMaxRequestBodyBytes + 1, s3err.ErrEntityTooLarge}, + // An s3vectors scope alone, on another route, keeps the 10 MiB limit. + {"s3vectors", 0, iamRequestBodyLimit + 1, s3err.ErrEntityTooLarge}, + {"iam", 0, iamRequestBodyLimit + 1, s3err.ErrEntityTooLarge}, + } { + t.Run(fmt.Sprintf("%s_route%d_%d", tc.service, tc.routeLimit, tc.size), func(t *testing.T) { + body := strings.Repeat("v", tc.size) + req, err := http.NewRequest("POST", "http://localhost:8333/PutVectors", strings.NewReader(body)) + assert.NoError(t, err) + if tc.routeLimit > 0 { + req = req.WithContext(s3_constants.SetRequestBodyLimitInContext(req.Context(), tc.routeLimit)) + } + _, err = signer.Sign(req, strings.NewReader(body), tc.service, "us-east-1", time.Now()) + assert.NoError(t, err) + + identity, _, errCode := iam.doesSignatureMatch(req) + assert.Equal(t, tc.want, errCode) + if tc.want == s3err.ErrNone { + assert.Equal(t, "vec", identity.Name) + got, err := io.ReadAll(req.Body) + assert.NoError(t, err) + assert.Equal(t, len(body), len(got)) + } + }) + } } // Benchmark streaming vs non-streaming memory usage diff --git a/weed/s3api/iceberg/commit_helpers.go b/weed/s3api/iceberg/commit_helpers.go index bd725d019d..8dd19818ec 100644 --- a/weed/s3api/iceberg/commit_helpers.go +++ b/weed/s3api/iceberg/commit_helpers.go @@ -282,6 +282,10 @@ func (s *Server) authorizeCommit(ctx context.Context, bucketARN string, namespac Name: tableName, }, identityName) }) + if err == nil { + bucketName, _ := s3tables.ParseBucketNameFromARN(bucketARN) + err = s.tablesManager.CheckVectorNamespaceWrite(ctx, s.filerClient, bucketName, namespace) + } if err == nil { return nil } diff --git a/weed/s3api/iceberg/handlers_table.go b/weed/s3api/iceberg/handlers_table.go index d99ba5a708..98debaeb12 100644 --- a/weed/s3api/iceberg/handlers_table.go +++ b/weed/s3api/iceberg/handlers_table.go @@ -308,6 +308,12 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) { writeManagerError(w, authErr) return } + // Registration confines the location to the table's namespace, but both + // branches write, and stage-create vends credentials, before it runs. + if err := s3tables.ValidateMetadataLocation(metadataLocation, bucketName, flattenNamespacePath(namespace)); err != nil { + writeError(w, http.StatusBadRequest, "BadRequestException", err.Error()) + return + } // Stage-create persists metadata in the internal staged area and skips S3Tables registration. if req.StageCreate { @@ -786,6 +792,9 @@ func (s *Server) mayUpdateTable(r *http.Request, bucketName string, namespace [] err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return s.tablesManager.AuthorizeUpdateTable(r.Context(), s3tables.NewManagerClient(client), updateReq, identityName) }) + if err == nil { + err = s.tablesManager.CheckVectorNamespaceWrite(r.Context(), s.filerClient, bucketName, namespace) + } if err != nil { glog.V(1).Infof("Iceberg: UpdateTable check for %s/%s failed, vending read-only: %v", bucketName, tableName, err) } diff --git a/weed/s3api/iceberg/vector_namespace_test.go b/weed/s3api/iceberg/vector_namespace_test.go new file mode 100644 index 0000000000..dad8e4ecb6 --- /dev/null +++ b/weed/s3api/iceberg/vector_namespace_test.go @@ -0,0 +1,130 @@ +package iceberg + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" +) + +type openAuthenticator struct{} + +func (openAuthenticator) AuthenticateRequest(*http.Request) (string, interface{}, s3err.ErrorCode) { + return s3tables.DefaultAccountID, nil, s3err.ErrNone +} + +func (openAuthenticator) DefaultAllow() bool { return true } + +// An index in a vector bucket is a catalog table the Iceberg catalog can load, +// but a caller holding every s3tables grant gets neither write credentials to +// it nor a commit into it. +func TestVectorIndexIsReadOnlyOverIceberg(t *testing.T) { + filer := s3tablestest.Start(t) + vendor := &stubVendor{credentials: &VendedCredentials{AccessKeyID: "ASIAEXAMPLE"}} + s := NewServer(s3tables.NewManagerClient(filer.Client), openAuthenticator{}) + s.SetCredentialVendor(vendor) + s.s3Endpoint = "http://127.0.0.1:8333" + router := mux.NewRouter().SkipClean(true) + s.RegisterRoutes(router) + + admin := s3tables.NewManager() + admin.SetTrusted(true) + client := s3tables.NewManagerClient(filer.Client) + arn, err := s3tables.BuildBucketARN(s3tables.DefaultRegion, s3tables.DefaultAccountID, "vec") + if err != nil { + t.Fatal(err) + } + metadata, _ := json.Marshal(map[string]any{"dimension": 4, "distanceMetric": "cosine"}) + for _, op := range []struct { + name string + req interface{} + }{ + {"CreateTableBucket", &s3tables.CreateTableBucketRequest{Name: "vec", Format: s3tables.FormatLance}}, + {"CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}}}, + {"CreateTable", &s3tables.CreateTableRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}, + Name: "idx", Format: s3tables.FormatLance, Metadata: &s3tables.TableMetadata{FullMetadata: metadata}}}, + } { + if err := admin.Execute(t.Context(), client, op.name, op.req, nil, ""); err != nil { + t.Fatalf("%s: %v", op.name, err) + } + } + + req := httptest.NewRequest(http.MethodGet, "/v1/vec/namespaces/vector/tables/idx", nil) + req.Header.Set(accessDelegationHeader, "vended-credentials") + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("load = %d (%s)", rec.Code, rec.Body.String()) + } + if vendor.gotPrefix != "vector/idx" || !vendor.gotReadOnly { + t.Errorf("vended prefix %q read-only %v, want vector/idx read-only", vendor.gotPrefix, vendor.gotReadOnly) + } + + reqErr := s.authorizeCommit(t.Context(), arn, []string{s3tables.VectorIndexNamespace}, "idx", s3tables.DefaultAccountID) + if reqErr == nil || reqErr.status != http.StatusForbidden { + t.Errorf("commit authorization = %+v, want 403", reqErr) + } +} + +// Stage-create registers nothing, so the catalog's own location and vector +// namespace checks never see it. It must not hand out write credentials to an +// index, whether the staged table names the index's location from another +// namespace or sits in the vector namespace itself. +func TestStageCreateCannotReachVectorIndex(t *testing.T) { + filer := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(filer.Client), openAuthenticator{}) + s.s3Endpoint = "http://127.0.0.1:8333" + router := mux.NewRouter().SkipClean(true) + s.RegisterRoutes(router) + + admin := s3tables.NewManager() + admin.SetTrusted(true) + client := s3tables.NewManagerClient(filer.Client) + arn, err := s3tables.BuildBucketARN(s3tables.DefaultRegion, s3tables.DefaultAccountID, "vec") + if err != nil { + t.Fatal(err) + } + for _, op := range []struct { + name string + req interface{} + }{ + {"CreateTableBucket", &s3tables.CreateTableBucketRequest{Name: "vec", Format: s3tables.FormatLance}}, + {"CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}}}, + {"CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{"other"}}}, + {"CreateTable", &s3tables.CreateTableRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}, + Name: "idx", Format: s3tables.FormatLance}}, + } { + if err := admin.Execute(t.Context(), client, op.name, op.req, nil, ""); err != nil { + t.Fatalf("%s: %v", op.name, err) + } + } + + for name, tc := range map[string]struct { + namespace, body string + status int + }{ + "index location from another namespace": {"other", `{"name":"x","stage-create":true,"location":"s3://vec/vector/idx"}`, http.StatusBadRequest}, + "vector namespace": {s3tables.VectorIndexNamespace, `{"name":"y","stage-create":true}`, http.StatusForbidden}, + } { + t.Run(name, func(t *testing.T) { + vendor := &stubVendor{credentials: &VendedCredentials{AccessKeyID: "ASIAEXAMPLE"}} + s.SetCredentialVendor(vendor) + req := httptest.NewRequest(http.MethodPost, "/v1/vec/namespaces/"+tc.namespace+"/tables", strings.NewReader(tc.body)) + req.Header.Set(accessDelegationHeader, "vended-credentials") + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != tc.status { + t.Errorf("stage-create = %d (%s), want %d", rec.Code, rec.Body.String(), tc.status) + } + if vendor.gotPrefix != "" && !vendor.gotReadOnly { + t.Errorf("vended write credentials to %q", vendor.gotPrefix) + } + }) + } +} diff --git a/weed/s3api/lance/cluster_token_test.go b/weed/s3api/lance/cluster_token_test.go new file mode 100644 index 0000000000..15409a3aa3 --- /dev/null +++ b/weed/s3api/lance/cluster_token_test.go @@ -0,0 +1,94 @@ +package lance + +import ( + "encoding/hex" + "net/http" + "net/http/httptest" + "testing" + "time" + + jwt "github.com/golang-jwt/jwt/v5" + "github.com/seaweedfs/seaweedfs/weed/security" +) + +func signAdminClaims(t *testing.T, key security.SigningKey, claims security.SeaweedFilerAdminClaims) string { + t.Helper() + token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(key)) + if err != nil { + t.Fatalf("sign: %v", err) + } + return token +} + +func trustedSeenByHandler(t *testing.T, s *Server, token string) bool { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil) + if token != "" { + req.Header.Set(clusterTokenHeader, token) + } + trusted := false + rec := httptest.NewRecorder() + s.Auth(func(w http.ResponseWriter, r *http.Request) { + trusted = isTrustedCaller(r.Context()) + w.WriteHeader(http.StatusOK) + })(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d (%s), want 200", rec.Code, rec.Body.String()) + } + return trusted +} + +func TestClusterTokenMarksTheCallerTrusted(t *testing.T) { + key := security.SigningKey("cluster-key") + s := NewServer(nil, openAuthenticator{}) + s.SetSigningKey(func() security.SigningKey { return key }) + + derived := clusterTokenKey(key) + expired := security.SeaweedFilerAdminClaims{} + expired.ExpiresAt = jwt.NewNumericDate(time.Now().Add(-time.Minute)) + for _, tc := range []struct { + name string + token string + want bool + }{ + {"no token", "", false}, + {"signed with the derived key", string(security.GenJwtForFilerAdmin(derived, 60)), true}, + {"signed with the cluster key itself", string(security.GenJwtForFilerAdmin(key, 60)), false}, + {"signed with another key", string(security.GenJwtForFilerAdmin(clusterTokenKey(security.SigningKey("other-key")), 60)), false}, + {"expired", signAdminClaims(t, derived, expired), false}, + {"session token", signAdminClaims(t, derived, security.SeaweedFilerAdminClaims{SessionId: "session"}), false}, + {"garbage", "not-a-token", false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := trustedSeenByHandler(t, s, tc.token); got != tc.want { + t.Errorf("trusted = %v, want %v", got, tc.want) + } + }) + } + + token := string(security.GenJwtForFilerAdmin(derived, 60)) + key = security.SigningKey("rotated-key") + if trustedSeenByHandler(t, s, token) { + t.Error("a token from before a key rotation is still trusted") + } + if !trustedSeenByHandler(t, s, string(security.GenJwtForFilerAdmin(clusterTokenKey(key), 60))) { + t.Error("a token signed after a key rotation is not trusted") + } + + key = nil + if trustedSeenByHandler(t, s, token) { + t.Error("a server with no cluster key trusted a caller") + } + s.SetSigningKey(nil) + if trustedSeenByHandler(t, s, token) { + t.Error("a server with no key source trusted a caller") + } +} + +// The Rust catalog client derives the same key; its tests pin this value too. +func TestClusterTokenKeyDerivation(t *testing.T) { + got := hex.EncodeToString(clusterTokenKey(security.SigningKey("cluster-key"))) + if want := "b38c0b66da41621f2a9b6de68e4239ddd0d2dd7b4ede01aa91bd34a17edc1021"; got != want { + t.Errorf("clusterTokenKey = %s, want %s", got, want) + } +} diff --git a/weed/s3api/lance/handlers_oauth_test.go b/weed/s3api/lance/handlers_oauth_test.go index 236b101f32..c16e47ce0e 100644 --- a/weed/s3api/lance/handlers_oauth_test.go +++ b/weed/s3api/lance/handlers_oauth_test.go @@ -49,8 +49,9 @@ func newTestServerWithOAuth() *Server { } type mockS3Authenticator struct { - called bool - errCode s3err.ErrorCode + called bool + errCode s3err.ErrorCode + authDisabled bool } func (m *mockS3Authenticator) AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode) { @@ -63,6 +64,8 @@ func (m *mockS3Authenticator) AuthenticateRequest(r *http.Request) (string, inte func (m *mockS3Authenticator) DefaultAllow() bool { return false } +func (m *mockS3Authenticator) AuthEnabled() bool { return !m.authDisabled } + func mintTestToken(t *testing.T, accessKey, secret string, issuedAt, expiresAt time.Time) string { t.Helper() key := deriveSigningKey(accessKey, secret) @@ -316,6 +319,32 @@ func TestAuthApiKeyInvalid(t *testing.T) { } } +// Without identities the S3 port takes any signature, so the catalog takes any +// key or token and leaves the caller to the S3 authenticator. +func TestAuthTakesAnyKeyWhenS3AuthIsOff(t *testing.T) { + s := newTestServerWithOAuth() + s.authenticator = &mockS3Authenticator{authDisabled: true} + + for header, value := range map[string]string{ + "x-api-key": "unknown:key", + "Authorization": "Bearer not-a-token", + } { + var gotIdentity string + handler := s.Auth(func(w http.ResponseWriter, r *http.Request) { + gotIdentity = s3_constants.GetIdentityNameFromContext(r) + w.WriteHeader(http.StatusOK) + }) + req := httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil) + req.Header.Set(header, value) + rec := httptest.NewRecorder() + handler(rec, req) + + if rec.Code != http.StatusOK || gotIdentity != "s3user" { + t.Fatalf("%s: status = %d, identity = %q, want 200 as s3user", header, rec.Code, gotIdentity) + } + } +} + func TestAuthNoBearerStillUsesS3Authenticator(t *testing.T) { s := newTestServerWithOAuth() auth := &mockS3Authenticator{errCode: s3err.ErrNone} diff --git a/weed/s3api/lance/handlers_table.go b/weed/s3api/lance/handlers_table.go index 65c0e1a11d..c2027d2d32 100644 --- a/weed/s3api/lance/handlers_table.go +++ b/weed/s3api/lance/handlers_table.go @@ -135,24 +135,35 @@ func (s *Server) handleDeclareTable(w http.ResponseWriter, r *http.Request) { return } - if err := s.createTable(r, bucket, ns, name, location); err != nil { + _, lookupErr := s.loadLanceTable(r, bucket, ns, name) + location, err = s.createTable(r, bucket, ns, name, location) + if err != nil { writeStorageError(w, r, err) return } - if err := s.writeMarker(r, location, reservedMarker); err != nil { - // The catalog entry is the authority; the marker only mirrors it for - // clients that read the storage prefix directly. - glog.V(1).Infof("lance: could not write %s for %s: %v", reservedMarker, location, err) + // A declare that creates the table may write it: a Lance commit is a direct + // S3 write that never revisits the catalog, so the create authorization is + // the write evidence. One that only finds the table answers like describe. + var writable bool + if isNotFound(lookupErr) { + writable = s.mayWriteNamespace(r, bucket, ns) + } else { + writable = s.mayUpdateTable(r, bucket, ns, name) } - // Declaring a name that was deregistered brings it back, the same way - // registering it does. - if err := s.removeMarker(r, location, deregisteredMarker); err != nil { - glog.V(2).Infof("lance: could not clear %s for %s: %v", deregisteredMarker, location, err) + if writable { + if err := s.writeMarker(r, location, reservedMarker); err != nil { + // The catalog entry is the authority; the marker only mirrors it for + // clients that read the storage prefix directly. + glog.V(1).Infof("lance: could not write %s for %s: %v", reservedMarker, location, err) + } + // Declaring a name that was deregistered brings it back, the same way + // registering it does. + if err := s.removeMarker(r, location, deregisteredMarker); err != nil { + glog.V(2).Infof("lance: could not clear %s for %s: %v", deregisteredMarker, location, err) + } } - // A Lance commit is a direct S3 write that never revisits the catalog, so - // the create authorization above is the write evidence for the dataset. - options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials), func() bool { return true }) + options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials), func() bool { return writable }) if err != nil { writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error()) return @@ -353,7 +364,7 @@ func (s *Server) handleRegisterTable(w http.ResponseWriter, r *http.Request) { } } default: - if err := s.createTable(r, bucket, ns, name, location); err != nil { + if location, err = s.createTable(r, bucket, ns, name, location); err != nil { writeStorageError(w, r, err) return } @@ -545,7 +556,9 @@ func (s *Server) loadLanceTable(r *http.Request, bucket string, ns []string, nam return &lanceTable{location: location, versionToken: resp.VersionToken}, nil } -func (s *Server) createTable(r *http.Request, bucket string, ns []string, name, location string) error { +// createTable returns the location the catalog holds for the table. When the +// name already exists that is its own location, not the one asked for. +func (s *Server) createTable(r *http.Request, bucket string, ns []string, name, location string) (string, error) { req := &s3tables.CreateTableRequest{ TableBucketARN: bucketARN(bucket), Namespace: ns, @@ -556,7 +569,13 @@ func (s *Server) createTable(r *http.Request, bucket string, ns []string, name, MetadataLocation: location, } var resp s3tables.CreateTableResponse - return s.execute(r, "CreateTable", req, &resp) + if err := s.execute(r, "CreateTable", req, &resp); err != nil { + return "", err + } + if resp.MetadataLocation == "" { + return tableLocation(bucket, ns, name), nil + } + return resp.MetadataLocation, nil } // repointTable moves an existing entry to another dataset location without diff --git a/weed/s3api/lance/handlers_test.go b/weed/s3api/lance/handlers_test.go index d62bdc7089..037f66ff83 100644 --- a/weed/s3api/lance/handlers_test.go +++ b/weed/s3api/lance/handlers_test.go @@ -24,6 +24,14 @@ func (openAuthenticator) AuthenticateRequest(*http.Request) (string, interface{} func (openAuthenticator) DefaultAllow() bool { return true } +func (openAuthenticator) AuthEnabled() bool { return false } + +// signedAuthenticator admits the same caller on a gateway that checks +// credentials. +type signedAuthenticator struct{ openAuthenticator } + +func (signedAuthenticator) AuthEnabled() bool { return true } + type testHarness struct { router *mux.Router filer *s3tablestest.MemFiler @@ -134,6 +142,28 @@ func decode[T any](t *testing.T, recorder *httptest.ResponseRecorder) T { return out } +// While the S3 port takes unsigned requests the catalog says so, and a client +// with no credentials of its own still reaches the data. Once auth is on it +// has to sign again. +func TestStorageOptionsSkipSignatureOnlyWhileS3AuthIsOff(t *testing.T) { + h := newTestHarness(t) + h.createBucket(t, "open", s3tables.FormatLance) + h.mustDo(t, http.MethodPost, "/v1/namespace/open$ns/create", `{}`, http.StatusOK) + + declared := decode[DeclareTableResponse](t, + h.mustDo(t, http.MethodPost, "/v1/table/open$ns$t/declare", `{}`, http.StatusOK)) + if declared.StorageOptions["aws_skip_signature"] != "true" { + t.Fatalf("auth off: storage options = %v, want aws_skip_signature", declared.StorageOptions) + } + + h.server.authenticator = signedAuthenticator{} + described := decode[DescribeTableResponse](t, + h.mustDo(t, http.MethodPost, "/v1/table/open$ns$t/describe", `{}`, http.StatusOK)) + if _, ok := described.StorageOptions["aws_skip_signature"]; ok { + t.Fatalf("auth on: storage options = %v, want no aws_skip_signature", described.StorageOptions) + } +} + // The lifecycle a Lance client drives: create the namespace, declare the table // before any data exists, resolve it to a location, then deregister and bring it // back by registering the same location. @@ -415,3 +445,48 @@ func TestDataPlaneIsUnsupported(t *testing.T) { t.Fatalf("error code = %d, want %d", got.Code, codeUnsupported) } } + +// createOnlyAuthenticator signs in an identity that may create and read tables +// but not update them. +type createOnlyAuthenticator struct{} + +func (createOnlyAuthenticator) AuthenticateRequest(*http.Request) (string, interface{}, s3err.ErrorCode) { + return "creator", &struct { + Name string + Actions []string + }{Name: "creator", Actions: []string{"s3tables:CreateTable", "s3tables:GetTable"}}, s3err.ErrNone +} + +func (createOnlyAuthenticator) DefaultAllow() bool { return false } + +func (createOnlyAuthenticator) AuthEnabled() bool { return true } + +// The right to create a name is the write evidence for a table the declare +// creates, but not for one it only finds. +func TestDeclareVendsWriteOnlyForATableItCreates(t *testing.T) { + h := newTestHarness(t) + h.createBucket(t, "lnc", s3tables.FormatLance) + h.mustDo(t, http.MethodPost, "/v1/namespace/lnc$ns/create", `{}`, http.StatusOK) + h.mustDo(t, http.MethodPost, "/v1/table/lnc$ns$theirs/declare", `{}`, http.StatusOK) + h.mustDo(t, http.MethodPost, "/v1/table/lnc$ns$theirs/deregister", `{}`, http.StatusOK) + + server := NewServer(s3tables.NewManagerClient(h.filer.Client), createOnlyAuthenticator{}) + server.SetCredentialVendor(recordingVendor{}) + router := mux.NewRouter().SkipClean(true) + server.RegisterRoutes(router) + creator := &testHarness{router: router, filer: h.filer, admin: h.admin, server: server} + + for _, tc := range []struct{ name, want string }{ + {"theirs", "read-only"}, + {"mine", "read-write"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := creator.vendedKey(t, "/v1/table/lnc$ns$"+tc.name+"/declare", ""); got != tc.want { + t.Errorf("vended %q, want %q", got, tc.want) + } + }) + } + if h.filer.Get("/buckets/lnc/ns/theirs", deregisteredMarker) == nil { + t.Error("a declare that may not write the table brought it back") + } +} diff --git a/weed/s3api/lance/server.go b/weed/s3api/lance/server.go index 53effee9e5..286205a70c 100644 --- a/weed/s3api/lance/server.go +++ b/weed/s3api/lance/server.go @@ -2,6 +2,8 @@ package lance import ( "context" + "crypto/hmac" + "crypto/sha256" "net/http" "strings" "time" @@ -12,6 +14,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/security" ) // FilerClient provides access to the filer for storage operations. @@ -22,6 +25,8 @@ type FilerClient interface { type S3Authenticator interface { AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode) DefaultAllow() bool + // AuthEnabled reports whether the S3 server checks credentials at all. + AuthEnabled() bool } // CredentialValidator validates S3 access key / secret key pairs @@ -61,6 +66,7 @@ type Server struct { credentialVendor CredentialVendor s3Endpoint string s3Region string + signingKey func() security.SigningKey } // NewServer creates a Lance namespace server over the given filer. @@ -89,6 +95,13 @@ func (s *Server) SetCredentialVendor(vendor CredentialVendor) { s.credentialVendor = vendor } +// SetSigningKey sets where to read the cluster key (jwt.filer_signing.key) +// that a trusted caller's clusterTokenHeader is checked against. It is read on +// every check, so a reloaded key takes effect at once. +func (s *Server) SetSigningKey(signingKey func() security.SigningKey) { + s.signingKey = signingKey +} + // SetS3Endpoint configures the S3 endpoint advertised in storage_options so a // client can reach the dataset without separately discovering the S3 address. func (s *Server) SetS3Endpoint(endpoint string) { @@ -166,17 +179,27 @@ func loggingMiddleware(next http.Handler) http.Handler { }) } +// authEnabled is false when the S3 server has no identities and takes any +// signature. The catalog then takes any key too, and leaves the caller to the +// S3 authenticator. +func (s *Server) authEnabled() bool { + return s.authenticator == nil || s.authenticator.AuthEnabled() +} + // Auth authenticates the caller and puts the identity in the request context. // Lance clients authenticate the catalog with a Bearer token or an x-api-key // header; the S3 authenticator stays for callers that can SigV4-sign. func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + if s.hasClusterToken(r) { + r = r.WithContext(withTrustedCaller(r.Context())) + } // A request carrying a Bearer token is a Lance REST client. An invalid // or expired token gets 401 immediately; falling through to the S3 // authenticator would parse the header as SigV4 and fail with a // different error that clients cannot act on. // The auth scheme is case-insensitive (RFC 7235). - if strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") { + if s.authEnabled() && strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") { if identityName, identity, accessKey, secretKey, ok := s.authenticateBearerKey(r); ok { ctx := r.Context() ctx = s3_constants.SetIdentityNameInContext(ctx, identityName) @@ -194,7 +217,7 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { return } - if apiKey := r.Header.Get("x-api-key"); apiKey != "" { + if apiKey := r.Header.Get("x-api-key"); apiKey != "" && s.authEnabled() { if identityName, identity, ok := s.authenticateApiKey(apiKey); ok { accessKey, secretKey, _ := strings.Cut(apiKey, ":") ctx := r.Context() @@ -250,3 +273,46 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { handler(w, r) } } + +// clusterTokenHeader carries a token signed with clusterTokenKey. The vector +// engine and the Lance maintenance worker send it next to their identity's +// credentials, and only they may write index data in a vector bucket. +const clusterTokenHeader = "X-Seaweedfs-Cluster-Token" + +// clusterTokenKey derives the key cluster tokens are signed with from the +// cluster key, so a token captured on its way here does not authorize the +// admin calls that take the cluster key itself. +func clusterTokenKey(key security.SigningKey) security.SigningKey { + mac := hmac.New(sha256.New, key) + mac.Write([]byte("seaweedfs-lance-cluster-token")) + return mac.Sum(nil) +} + +type trustedCallerKey struct{} + +func withTrustedCaller(ctx context.Context) context.Context { + return context.WithValue(ctx, trustedCallerKey{}, true) +} + +func isTrustedCaller(ctx context.Context) bool { + trusted, _ := ctx.Value(trustedCallerKey{}).(bool) + return trusted +} + +// hasClusterToken reports whether the request proves it holds the cluster key. +// Without a configured key no caller can. +func (s *Server) hasClusterToken(r *http.Request) bool { + token := r.Header.Get(clusterTokenHeader) + if token == "" || s.signingKey == nil { + return false + } + key := s.signingKey() + if len(key) == 0 { + return false + } + if security.IsFilerAdminToken(clusterTokenKey(key), security.EncodedJwt(token)) { + return true + } + glog.V(1).Infof("lance: %s from %s is not signed with a key derived from jwt.filer_signing.key", clusterTokenHeader, r.RemoteAddr) + return false +} diff --git a/weed/s3api/lance/storage.go b/weed/s3api/lance/storage.go index c4ccd67385..e719680118 100644 --- a/weed/s3api/lance/storage.go +++ b/weed/s3api/lance/storage.go @@ -149,6 +149,11 @@ func (s *Server) storageOptions(r *http.Request, bucket, location string, vend b if s.s3Region != "" { options["aws_region"] = s.s3Region } + if !s.authEnabled() { + // The S3 port takes unsigned requests, and a client with no + // credentials of its own would otherwise fail looking for some. + options["aws_skip_signature"] = "true" + } if !vend || s.credentialVendor == nil { return options, nil @@ -177,6 +182,9 @@ func (s *Server) storageOptions(r *http.Request, bucket, location string, vend b // credentials would cover, the same gate the Iceberg catalog applies so a // read-only caller never gets write-capable storage_options. func (s *Server) mayUpdateTable(r *http.Request, bucket string, ns []string, name string) bool { + if !s.mayWriteNamespace(r, bucket, ns) { + return false + } req := &s3tables.UpdateTableRequest{ TableBucketARN: bucketARN(bucket), Namespace: ns, @@ -192,6 +200,20 @@ func (s *Server) mayUpdateTable(r *http.Request, bucket string, ns []string, nam return err == nil } +// mayWriteNamespace keeps the vector namespace of a vector bucket out of reach +// of s3tables grants: its index data is written only by trusted callers, and +// the vector bucket policy gates everyone else through S3 Vectors. +func (s *Server) mayWriteNamespace(r *http.Request, bucket string, ns []string) bool { + if isTrustedCaller(r.Context()) { + return true + } + err := s.tablesManager.CheckVectorNamespaceWrite(r.Context(), s.filerClient, bucket, ns) + if err != nil { + glog.V(1).Infof("lance: %s/%s is read-only to this caller: %v", bucket, strings.Join(ns, "."), err) + } + return err == nil +} + // locationPrefix strips the s3://bucket/ part of a location, leaving the key // prefix a credential is scoped to. func locationPrefix(location string) string { diff --git a/weed/s3api/lance/vector_namespace_test.go b/weed/s3api/lance/vector_namespace_test.go index 01ec1770a8..3185684df3 100644 --- a/weed/s3api/lance/vector_namespace_test.go +++ b/weed/s3api/lance/vector_namespace_test.go @@ -1,10 +1,14 @@ package lance import ( + "context" "net/http" + "net/http/httptest" + "strings" "testing" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/security" ) // seedVectorIndex creates bucket/vector/idx through the trusted path, the way @@ -75,3 +79,87 @@ func TestVectorNamespaceOfUndeclaredBucketIsWritableOverLance(t *testing.T) { h.mustDo(t, http.MethodPost, idx+"/register", `{"location":"s3://old/vector/idx"}`, http.StatusOK) h.mustDo(t, http.MethodPost, idx+"/drop", `{}`, http.StatusOK) } + +// recordingVendor vends a credential whose key says whether it may write and +// whose secret is the prefix it covers. +type recordingVendor struct{} + +func (recordingVendor) VendTableCredentials(_ context.Context, _, _, prefix string, readOnly bool) (*VendedCredentials, error) { + if readOnly { + return &VendedCredentials{AccessKeyID: "read-only", SecretAccessKey: prefix}, nil + } + return &VendedCredentials{AccessKeyID: "read-write", SecretAccessKey: prefix}, nil +} + +func (h *testHarness) vendedKey(t *testing.T, target, token string) string { + t.Helper() + req := httptest.NewRequest(http.MethodPost, target, strings.NewReader(`{"vend_credentials":true}`)) + if token != "" { + req.Header.Set(clusterTokenHeader, token) + } + rec := httptest.NewRecorder() + h.router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("POST %s = %d (%s)", target, rec.Code, rec.Body.String()) + } + return decode[DescribeTableResponse](t, rec).StorageOptions["aws_access_key_id"] +} + +// A caller holding every s3tables grant still gets read-only credentials to a +// vector bucket's index data; only the cluster's own components can write it. +func TestVectorNamespaceVendsWriteCredentialsOnlyToTrustedCallers(t *testing.T) { + h := newTestHarness(t) + key := security.SigningKey("cluster-key") + h.server.SetSigningKey(func() security.SigningKey { return key }) + h.server.SetCredentialVendor(recordingVendor{}) + h.seedVectorIndex(t, "vec", s3tables.FormatLance) + h.mustDo(t, http.MethodPost, "/v1/table/vec$other$tbl/declare", `{}`, http.StatusOK) + h.seedVectorIndex(t, "old", "") + + h.vendedKey(t, "/v1/table/vec$vector$idx/declare", "") + if h.filer.Get("/buckets/vec/vector/idx", reservedMarker) != nil { + t.Error("an untrusted declare wrote into the index") + } + + trusted := string(security.GenJwtForFilerAdmin(clusterTokenKey(key), 60)) + forged := string(security.GenJwtForFilerAdmin(key, 60)) + for _, tc := range []struct { + name, target, token, want string + }{ + {"describe the index", "/v1/table/vec$vector$idx/describe", "", "read-only"}, + {"declare the index", "/v1/table/vec$vector$idx/declare", "", "read-only"}, + {"describe with an admin token", "/v1/table/vec$vector$idx/describe", forged, "read-only"}, + {"trusted describe", "/v1/table/vec$vector$idx/describe", trusted, "read-write"}, + {"trusted declare", "/v1/table/vec$vector$idx/declare", trusted, "read-write"}, + {"another namespace", "/v1/table/vec$other$tbl/describe", "", "read-write"}, + {"bucket from before formats", "/v1/table/old$vector$idx/describe", "", "read-write"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := h.vendedKey(t, tc.target, tc.token); got != tc.want { + t.Errorf("vended %q, want %q", got, tc.want) + } + }) + } +} + +// Declaring a name that already exists answers for that table, so naming the +// index's location on another table gets no credentials to the index. +func TestDeclareOfAnExistingTableKeepsItsLocation(t *testing.T) { + h := newTestHarness(t) + h.server.SetCredentialVendor(recordingVendor{}) + h.seedVectorIndex(t, "vec", s3tables.FormatLance) + h.mustDo(t, http.MethodPost, "/v1/table/vec$other$tbl/declare", `{}`, http.StatusOK) + + rec := h.mustDo(t, http.MethodPost, "/v1/table/vec$other$tbl/declare", + `{"location":"s3://vec/vector/idx","vend_credentials":true}`, http.StatusOK) + resp := decode[DeclareTableResponse](t, rec) + if resp.Location != "s3://vec/other/tbl" { + t.Errorf("location = %q, want the table's own s3://vec/other/tbl", resp.Location) + } + if got := resp.StorageOptions["aws_secret_access_key"]; got != "other/tbl" { + t.Errorf("credentials cover %q, want other/tbl", got) + } + if h.filer.Get("/buckets/vec/vector/idx", reservedMarker) != nil { + t.Error("the declare wrote into the index") + } +} diff --git a/weed/s3api/s3_bucket_encryption.go b/weed/s3api/s3_bucket_encryption.go index a1e647750c..b5ba8c6b54 100644 --- a/weed/s3api/s3_bucket_encryption.go +++ b/weed/s3api/s3_bucket_encryption.go @@ -213,12 +213,28 @@ func (s3a *S3ApiServer) getEncryptionConfiguration(bucket string) (*s3_pb.Encryp } if metadata.Encryption == nil { - return nil, s3err.ErrNoSuchBucketEncryptionConfiguration + return s3a.declaredEncryption(bucket) } return metadata.Encryption, s3err.ErrNone } +// declaredEncryption is the encryption a table bucket's record declares. It +// applies while the bucket has no S3 default encryption of its own. +func (s3a *S3ApiServer) declaredEncryption(bucket string) (*s3_pb.EncryptionConfiguration, s3err.ErrorCode) { + if s3a.bucketConfigCache == nil { + return nil, s3err.ErrNoSuchBucketEncryptionConfiguration + } + config, errCode := s3a.getBucketConfig(bucket) + if errCode != s3err.ErrNone { + return nil, errCode + } + if config.DeclaredEncryption == nil { + return nil, s3err.ErrNoSuchBucketEncryptionConfiguration + } + return config.DeclaredEncryption, s3err.ErrNone +} + // updateEncryptionConfiguration updates the encryption configuration for a bucket func (s3a *S3ApiServer) updateEncryptionConfiguration(bucket string, encryptionConfig *s3_pb.EncryptionConfiguration) s3err.ErrorCode { // Update using structured API @@ -249,6 +265,10 @@ func (s3a *S3ApiServer) removeEncryptionConfiguration(bucket string) s3err.Error } if metadata.Encryption == nil { + // A bucket's declared encryption is fixed when the bucket is created. + if declared, _ := s3a.declaredEncryption(bucket); declared != nil { + return s3err.ErrMethodNotAllowed + } return s3err.ErrNoSuchBucketEncryptionConfiguration } diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index a49d40220e..6256d4376b 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -359,6 +359,7 @@ const ( contextKeyOriginalRequestPath contextKey = "s3-original-request-path" contextKeyAccessKeyId contextKey = "s3-access-key-id" contextKeyStoredAccessKeyId contextKey = "s3-stored-access-key-id" + contextKeyRequestBodyLimit contextKey = "s3-request-body-limit" ) // identityHolder is a mutable container for the authenticated identity name, @@ -529,6 +530,20 @@ func GetOriginalRequestPathFromContext(r *http.Request) string { return "" } +// SetRequestBodyLimitInContext sets how much of the body SigV4 verification +// reads. The route sets it: the credential scope is not verified yet. +func SetRequestBodyLimitInContext(ctx context.Context, limit int64) context.Context { + return context.WithValue(ctx, contextKeyRequestBodyLimit, limit) +} + +// GetRequestBodyLimitFromContext returns the route's limit, or def. +func GetRequestBodyLimitFromContext(r *http.Request, def int64) int64 { + if limit, ok := r.Context().Value(contextKeyRequestBodyLimit).(int64); ok { + return limit + } + return def +} + // SetAccessKeyIdInContext stores the access key id the request authenticated // with: a static key's id, or an STS session's temporary key id. Several keys // can belong to one identity, so the identity name alone cannot tell their diff --git a/weed/s3api/s3_constants/s3_config.go b/weed/s3api/s3_constants/s3_config.go index d2d2c257aa..ab34ce39af 100644 --- a/weed/s3api/s3_constants/s3_config.go +++ b/weed/s3api/s3_constants/s3_config.go @@ -13,6 +13,9 @@ var ( Separator = ":" ) +// S3VectorsMaxRequestBodyBytes caps the body of an S3 Vectors API request. +const S3VectorsMaxRequestBodyBytes = 32 << 20 + func Concat(elements ...string) string { return strings.Join(elements, Separator) } diff --git a/weed/s3api/s3_sse_s3.go b/weed/s3api/s3_sse_s3.go index 8ce3542d28..32c386e428 100644 --- a/weed/s3api/s3_sse_s3.go +++ b/weed/s3api/s3_sse_s3.go @@ -778,6 +778,13 @@ func (km *SSES3KeyManager) StoreKey(key *SSES3Key) { // The DEK is encrypted with the super key and stored in object metadata } +// IsConfigured reports whether a KEK is loaded, i.e. SSE-S3 can encrypt. +func (km *SSES3KeyManager) IsConfigured() bool { + km.mu.RLock() + defer km.mu.RUnlock() + return len(km.superKey) > 0 +} + // GetMasterKey returns a derived key from the master KEK for STS signing // This uses HKDF to isolate the STS security domain from the SSE-S3 domain func (km *SSES3KeyManager) GetMasterKey() []byte { diff --git a/weed/s3api/s3api_bucket_config.go b/weed/s3api/s3api_bucket_config.go index 254b2adb83..427e58dcf5 100644 --- a/weed/s3api/s3api_bucket_config.go +++ b/weed/s3api/s3api_bucket_config.go @@ -24,6 +24,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" ) // BucketConfig represents cached bucket configuration. Only fields parsed @@ -56,6 +57,7 @@ type BucketConfig struct { LifecycleTransitionMinSize string Tags map[string]string // parsed from entry content Encryption *s3_pb.EncryptionConfiguration // parsed from entry content + DeclaredEncryption *s3_pb.EncryptionConfiguration // declared by a table bucket's record KMSKeyCache *BucketKMSCache // Per-bucket KMS key cache for SSE-KMS operations LastModified time.Time } @@ -441,6 +443,9 @@ func (s3a *S3ApiServer) newBucketConfigFromEntry(bucket string, entry *filer_pb. config.ObjectLockConfig = objectLockConfig } config.BucketPolicy = loadBucketPolicyFromExtended(entry, bucket) + if algorithm := s3tables.DeclaredBucketEncryption(entry); algorithm != "" { + config.DeclaredEncryption = &s3_pb.EncryptionConfiguration{SseAlgorithm: algorithm} + } if lifecycleXML, exists := entry.Extended[bucketLifecycleConfigurationXMLKey]; exists && len(lifecycleXML) > 0 { config.LifecycleXML = lifecycleXML diff --git a/weed/s3api/s3api_bucket_default_encryption_test.go b/weed/s3api/s3api_bucket_default_encryption_test.go index 74484593f9..1ac7b19447 100644 --- a/weed/s3api/s3api_bucket_default_encryption_test.go +++ b/weed/s3api/s3api_bucket_default_encryption_test.go @@ -2,6 +2,7 @@ package s3api import ( "bytes" + "errors" "net/http" "net/http/httptest" "strings" @@ -12,6 +13,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" ) // Issue 11647: Bucket default encryption (SSE-S3) must be applied when @@ -218,3 +220,72 @@ func TestPutObjectUnderUploadsFolderKeepsBucketDefaultEncryption(t *testing.T) { } } } + +// A vector bucket is a LANCE table bucket; the AES256 its record declares is +// the bucket's default encryption. An ICEBERG bucket's declaration is not +// applied, because its catalog reads client manifests straight from the filer. +func TestPutObjectAppliesTableBucketDeclaredEncryption(t *testing.T) { + km := GetSSES3KeyManager() + oldSuperKey := km.superKey + km.superKey = make([]byte, 32) + for i := range km.superKey { + km.superKey[i] = byte(i + 1) + } + t.Cleanup(func() { + km.superKey = oldSuperKey + }) + + volume := startFakeVolumeServer(t) + filerImpl := &ambiguousPutFiler{ + volume: volume, + entries: map[string]*filer_pb.Entry{}, + apply: true, + } + s3a := newPutTestServer(t, startFakeFiler(t, filerImpl)) + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + for bucket, format := range map[string]string{"lance": s3tables.FormatLance, "ice": s3tables.FormatIceberg} { + s3a.bucketConfigCache.Set(bucket, s3a.newBucketConfigFromEntry(bucket, &filer_pb.Entry{ + Name: bucket, + Extended: map[string][]byte{ + s3tables.ExtendedKeyTableBucket: []byte("true"), + s3tables.ExtendedKeyMetadata: []byte(`{"name":"` + bucket + `","format":"` + format + + `","encryptionConfiguration":{"sseAlgorithm":"AES256"}}`), + }, + })) + } + + cfg, err := s3a.GetBucketEncryptionConfig("lance") + if err != nil || cfg.SseAlgorithm != EncryptionTypeAES256 { + t.Fatalf("GetBucketEncryptionConfig(lance) = %v, %v; want AES256", cfg, err) + } + if metadata, err := s3a.GetBucketMetadata("lance"); err != nil || metadata.Encryption != nil { + t.Fatalf("declared encryption leaked into the stored bucket metadata: %v, %v", metadata, err) + } + if _, err := s3a.GetBucketEncryptionConfig("ice"); !errors.Is(err, ErrNoEncryptionConfig) { + t.Fatalf("GetBucketEncryptionConfig(ice) error = %v, want ErrNoEncryptionConfig", err) + } + if code := s3a.removeEncryptionConfiguration("lance"); code != s3err.ErrMethodNotAllowed { + t.Fatalf("DeleteBucketEncryption(lance) = %v, want MethodNotAllowed for a declared encryption", code) + } + if code := s3a.removeEncryptionConfiguration("ice"); code != s3err.ErrNoSuchBucketEncryptionConfiguration { + t.Fatalf("DeleteBucketEncryption(ice) = %v, want NoSuchBucketEncryptionConfiguration", code) + } + + for bucket, want := range map[string]filer_pb.SSEType{"lance": filer_pb.SSEType_SSE_S3, "ice": filer_pb.SSEType_NONE} { + filePath := "/buckets/" + bucket + "/vector/idx/data/0.lance" + r := httptest.NewRequest(http.MethodPut, "/"+bucket+"/vector/idx/data/0.lance", nil) + _, code, _ := s3a.putToFiler(r, filePath, strings.NewReader("lance data file"), bucket, "vector/idx/data/0.lance", 1, 0, nil, false, "") + if code != s3err.ErrNone { + t.Fatalf("putToFiler(%s) = %v, want %v", bucket, code, s3err.ErrNone) + } + entry := filerImpl.entries[filePath] + if entry == nil || len(entry.Chunks) == 0 { + t.Fatalf("no entry with chunks at %s", filePath) + } + for i, chunk := range entry.Chunks { + if chunk.SseType != want { + t.Errorf("%s chunk %d: SseType %v, want %v", bucket, i, chunk.SseType, want) + } + } + } +} diff --git a/weed/s3api/s3api_embedded_iam.go b/weed/s3api/s3api_embedded_iam.go index 1c6aa1bec0..6b5e244469 100644 --- a/weed/s3api/s3api_embedded_iam.go +++ b/weed/s3api/s3api_embedded_iam.go @@ -342,6 +342,8 @@ func iamAuthErrorCode(errCode s3err.ErrorCode) string { return "RequestTimeTooSkewed" case s3err.ErrInternalError: return iam.ErrCodeServiceFailureException + case s3err.ErrEntityTooLarge: + return "InvalidInput" default: return "IncompleteSignature" } diff --git a/weed/s3api/s3api_embedded_iam_test.go b/weed/s3api/s3api_embedded_iam_test.go index 4269958de9..a7043b7114 100644 --- a/weed/s3api/s3api_embedded_iam_test.go +++ b/weed/s3api/s3api_embedded_iam_test.go @@ -3041,5 +3041,6 @@ func TestIamAuthErrorCode(t *testing.T) { assert.Equal(t, "SignatureDoesNotMatch", iamAuthErrorCode(s3err.ErrSignatureDoesNotMatch)) assert.Equal(t, "AccessDenied", iamAuthErrorCode(s3err.ErrAccessDenied)) assert.Equal(t, "RequestTimeTooSkewed", iamAuthErrorCode(s3err.ErrRequestTimeTooSkewed)) + assert.Equal(t, "InvalidInput", iamAuthErrorCode(s3err.ErrEntityTooLarge)) assert.Equal(t, "IncompleteSignature", iamAuthErrorCode(s3err.ErrMissingFields)) } diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 8bd16cb1ec..dfc2c41ab4 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -1106,9 +1106,9 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // GetBucketLifecycleConfiguration bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLifecycleConfigurationHandler, ACTION_READ)), "GET")).Queries("lifecycle", "") // PutBucketLifecycleConfiguration - bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketLifecycleConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("lifecycle", "") + bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketLifecycleConfigurationHandler, ACTION_ADMIN)), "PUT")).Queries("lifecycle", "") // DeleteBucketLifecycleConfiguration - bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketLifecycleHandler, ACTION_WRITE)), "DELETE")).Queries("lifecycle", "") + bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketLifecycleHandler, ACTION_ADMIN)), "DELETE")).Queries("lifecycle", "") // GetBucketLocation bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLocationHandler, ACTION_READ)), "GET")).Queries("location", "") @@ -1500,6 +1500,21 @@ func (s3a *S3ApiServer) AuthenticateRequest(r *http.Request) (string, interface{ return "", nil, err } +// FilerSigningKey is the current jwt.filer_signing.key, which follows a +// security.toml reload. +func (s3a *S3ApiServer) FilerSigningKey() security.SigningKey { + if s3a.filerGuard == nil { + return nil + } + return s3a.filerGuard.SigningKey() +} + +// AuthEnabled reports whether requests must carry valid credentials. With no +// identities configured, every request is allowed as the admin. +func (s3a *S3ApiServer) AuthEnabled() bool { + return s3a.iam == nil || s3a.iam.isEnabled() +} + // DefaultAllow returns whether access is allowed by default when no policy is found func (s3a *S3ApiServer) DefaultAllow() bool { if s3a.iam == nil || s3a.iam.iamIntegration == nil { diff --git a/weed/s3api/s3api_server_routing_test.go b/weed/s3api/s3api_server_routing_test.go index 9316174924..b008b18d99 100644 --- a/weed/s3api/s3api_server_routing_test.go +++ b/weed/s3api/s3api_server_routing_test.go @@ -5,6 +5,8 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" + "path/filepath" "strings" "testing" "time" @@ -529,3 +531,60 @@ func TestRouting_BucketRootPreflightUsesBucketCORS(t *testing.T) { }) } } + +// Write covers a bucket's objects, not its configuration: an identity holding +// Write on the bucket is refused lifecycle writes, while Admin on the bucket or +// a policy granting s3:PutLifecycleConfiguration passes. The bucket is +// negatively cached, so a request that gets past authorization ends at the +// handler's NoSuchBucket. +func TestRouting_BucketLifecycleWritesNeedAdmin(t *testing.T) { + const bucket = "config-bucket" + configFile := filepath.Join(t.TempDir(), "s3.json") + require.NoError(t, os.WriteFile(configFile, []byte(`{ + "identities": [ + {"name":"writer","credentials":[{"accessKey":"WRITERKEY","secretKey":"writersecret"}],"actions":["Read:`+bucket+`","Write:`+bucket+`","List:`+bucket+`","Tagging:`+bucket+`"]}, + {"name":"bucketadmin","credentials":[{"accessKey":"BUCKETADMINKEY","secretKey":"bucketadminsecret"}],"actions":["Admin:`+bucket+`"]}, + {"name":"lifecycler","credentials":[{"accessKey":"LIFECYCLERKEY","secretKey":"lifecyclersecret"}],"policyNames":["LifecyclePolicy"]} + ], + "policies": [ + {"name":"LifecyclePolicy","content":"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:PutLifecycleConfiguration\"],\"Resource\":[\"arn:aws:s3:::`+bucket+`\"]}]}"} + ] +}`), 0600)) + + s3a := setupRoutingTestServer(t) + s3a.iam = NewIdentityAccessManagementWithStore(&S3ApiServerOption{Config: configFile}, nil, "memory") + require.True(t, s3a.iam.isEnabled()) + s3a.cb = &CircuitBreaker{s3a: s3a} + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.SetNegativeCache(bucket) + router := mux.NewRouter() + s3a.registerRouter(router) + + send := func(accessKey, secretKey, method, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, "http://localhost:8333/"+bucket+"?lifecycle", strings.NewReader(body)) + creds := credentials.NewStaticCredentials(accessKey, secretKey, "") + _, err := v4.NewSigner(creds).Sign(req, strings.NewReader(body), "s3", "us-east-1", time.Now()) + require.NoError(t, err) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + return rr + } + + const lifecycleXML = `rEnabledp/1` + for _, tc := range []struct { + method, body string + }{ + {http.MethodPut, lifecycleXML}, + {http.MethodDelete, ""}, + } { + rr := send("WRITERKEY", "writersecret", tc.method, tc.body) + assert.Equal(t, http.StatusForbidden, rr.Code, "writer %s", tc.method) + assert.Contains(t, rr.Body.String(), "AccessDenied", "writer %s", tc.method) + + for _, key := range [][2]string{{"BUCKETADMINKEY", "bucketadminsecret"}, {"LIFECYCLERKEY", "lifecyclersecret"}} { + rr = send(key[0], key[1], tc.method, tc.body) + assert.Equal(t, http.StatusNotFound, rr.Code, "%s %s", key[0], tc.method) + assert.Contains(t, rr.Body.String(), "NoSuchBucket", "%s %s", key[0], tc.method) + } + } +} diff --git a/weed/s3api/s3api_tables.go b/weed/s3api/s3api_tables.go index 5a382d0f3f..6b50017197 100644 --- a/weed/s3api/s3api_tables.go +++ b/weed/s3api/s3api_tables.go @@ -58,6 +58,11 @@ func (st *S3TablesApiServer) SetLocalAccountChecker(fn func(string) bool) { st.handler.SetLocalAccountChecker(fn) } +// SetEncryptionChecker sets the handler's SSE-S3 availability predicate. +func (st *S3TablesApiServer) SetEncryptionChecker(fn func() bool) { + st.handler.SetEncryptionChecker(fn) +} + // S3TablesHandler handles S3 Tables API requests func (st *S3TablesApiServer) S3TablesHandler(w http.ResponseWriter, r *http.Request) { st.handler.HandleRequest(w, r, st) @@ -72,6 +77,7 @@ func (st *S3TablesApiServer) WithFilerClient(streamingMode bool, fn func(filer_p func (s3a *S3ApiServer) registerS3TablesRoutes(router *mux.Router) { // Create S3 Tables handler s3TablesApi := NewS3TablesApiServer(s3a) + s3TablesApi.SetEncryptionChecker(GetSSES3KeyManager().IsConfigured) if s3a.iam != nil { s3TablesApi.SetLocalAccountChecker(s3a.iam.isConfiguredAccountID) } diff --git a/weed/s3api/s3api_tables_auth_test.go b/weed/s3api/s3api_tables_auth_test.go index 6ab18d59d1..85080f5a69 100644 --- a/weed/s3api/s3api_tables_auth_test.go +++ b/weed/s3api/s3api_tables_auth_test.go @@ -159,6 +159,34 @@ func TestLanceAuthRejectsFailedSignature(t *testing.T) { assert.NotEqual(t, http.StatusOK, rr.Code) } +// The Lance catalog takes any x-api-key while the S3 port runs without +// identities, as the admin, and refuses an unknown key once auth is on. +func TestLanceAuthTakesAnyKeyOnlyWhileS3AuthIsOff(t *testing.T) { + s3a := &S3ApiServer{iam: &IdentityAccessManagement{}} + server := lance.NewServer(nil, s3a) + server.SetCredentialValidator(s3a) + + call := func() (int, string) { + req := httptest.NewRequest(http.MethodGet, "/v1/namespace/list", nil) + req.Header.Set("x-api-key", "unknown:key") + identity := "" + rr := httptest.NewRecorder() + server.Auth(func(w http.ResponseWriter, r *http.Request) { + identity = s3_constants.GetIdentityNameFromContext(r) + w.WriteHeader(http.StatusOK) + })(rr, req) + return rr.Code, identity + } + + code, identity := call() + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, s3_constants.AccountAdminId, identity) + + s3a.iam.isAuthEnabled = true + code, _ = call() + assert.Equal(t, http.StatusUnauthorized, code) +} + func TestAuthSignatureOnlyScrubsAccountHeader(t *testing.T) { s3a := setupDefaultAllowAuthServer(t) diff --git a/weed/s3api/s3tables/filer_ops.go b/weed/s3api/s3tables/filer_ops.go index 96fae0a259..80ad13ef25 100644 --- a/weed/s3api/s3tables/filer_ops.go +++ b/weed/s3api/s3tables/filer_ops.go @@ -11,8 +11,10 @@ import ( "path" "time" + "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -107,6 +109,20 @@ func (h *S3TablesHandler) mutateEntry( client filer_pb.SeaweedFilerClient, path string, mutate func(entry *filer_pb.Entry) error, +) error { + return h.mutateEntryAsserting(ctx, client, path, nil, mutate) +} + +// mutateEntryAsserting is mutateEntry with the keys in absent asserted at the +// write too, as this package's own attributes are: one absent when the entry was +// read and written by someone else since fails the compare, instead of being +// written over with the entry as it was read. +func (h *S3TablesHandler) mutateEntryAsserting( + ctx context.Context, + client filer_pb.SeaweedFilerClient, + path string, + absent []string, + mutate func(entry *filer_pb.Entry) error, ) error { dir, name := splitPath(path) @@ -120,7 +136,7 @@ func (h *S3TablesHandler) mutateEntry( } entry := resp.Entry - expected := snapshotExtended(entry.Extended) + expected := snapshotExtended(entry.Extended, absent...) if entry.Extended == nil { entry.Extended = make(map[string][]byte) } @@ -401,13 +417,25 @@ func (h *S3TablesHandler) getExtendedAttribute(ctx context.Context, client filer // readNamespaceMetadataAttr returns the namespace-metadata xattr at // namespacePath. A directory without it is not a namespace — a delete that kept -// the data under it leaves one — so its absence reads as a missing entry. +// the data under it leaves one — so its absence reads as a missing entry. So +// does a namespace still marked by its create: like a table's, it is absent to +// every reader until the create confirms it. func (h *S3TablesHandler) readNamespaceMetadataAttr(ctx context.Context, client filer_pb.SeaweedFilerClient, namespacePath string) ([]byte, error) { - data, err := h.getExtendedAttribute(ctx, client, namespacePath, ExtendedKeyMetadata) - if errors.Is(err, ErrAttributeNotFound) { + entry, err := h.lookupEntry(ctx, client, namespacePath) + if err != nil { + return nil, err + } + data, ok := entry.Extended[ExtendedKeyMetadata] + if !ok || isUnfinishedCreate(entry.Extended) { return nil, fmt.Errorf("%w: %s", filer_pb.ErrNotFound, namespacePath) } - return data, err + return data, nil +} + +// isFinishedNamespaceEntry is isNamespaceEntry for a namespace its create has +// confirmed, the only kind a reader or a delete may act on. +func isFinishedNamespaceEntry(entry *filer_pb.Entry) bool { + return isNamespaceEntry(entry) && !isUnfinishedCreate(entry.Extended) } // loadNamespaceMetadata resolves a namespace to its metadata. A directory that @@ -455,14 +483,6 @@ func (h *S3TablesHandler) lookupEntry(ctx context.Context, client filer_pb.Seawe return resp.Entry, nil } -// deleteExtendedAttribute deletes an extended attribute from an entry -func (h *S3TablesHandler) deleteExtendedAttribute(ctx context.Context, client filer_pb.SeaweedFilerClient, path, key string) error { - return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error { - delete(extended, key) - return nil - }) -} - // deleteDirectory deletes a directory and all its contents func (h *S3TablesHandler) deleteDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error { dir, name := splitPath(path) @@ -499,9 +519,46 @@ func (h *S3TablesHandler) deleteDirectoryIfMatches(ctx context.Context, client f return nil } +// errDirNotEmpty is a directory that holds something the caller meant to leave +// alone: a file in a leftover tree, or an entry the filer found in a directory +// it was asked to delete without its contents. +var errDirNotEmpty = errors.New("directory is not empty") + +// deleteEmptyDirectoryIfMatches deletes a directory without its contents, only +// while it still carries the extended attributes the caller captured. The +// filer checks both under the path lock, in the one delete: a changed +// attribute is ErrConcurrentUpdate, and an entry that appeared in the +// directory is errDirNotEmpty, so neither a directory taken back since the +// caller looked at it nor a file written into it since is lost. The mark of an +// empty S3 object stored on the directory is asserted absent too, so one +// written since is kept the same way. +func (h *S3TablesHandler) deleteEmptyDirectoryIfMatches(ctx context.Context, client filer_pb.SeaweedFilerClient, path string, expected *filer_pb.Entry) error { + dir, name := splitPath(path) + resp, err := client.DeleteEntry(ctx, &filer_pb.DeleteEntryRequest{ + Directory: dir, + Name: name, + IsDeleteData: true, + ExpectedExtended: snapshotExtended(expected.Extended, s3_constants.SeaweedFSPrefixObject), + }) + if status.Code(err) == codes.FailedPrecondition { + return fmt.Errorf("%w: %s", ErrConcurrentUpdate, path) + } + if err != nil { + return err + } + if resp != nil && resp.Error != "" { + if err := filer.DeleteEntryError(resp.Error); !errors.Is(err, filer.ErrNonEmptyFolder) { + return err + } + return fmt.Errorf("%w: %s", errDirNotEmpty, path) + } + return nil +} + var ( errParentDeleting = errors.New("parent is being deleted") errParentGone = errors.New("parent was deleted") + errParentCreating = errors.New("parent is still being created") errNoDeleteClaim = errors.New("claim not held") errBackOutFailed = errors.New("backing out the create failed") errSourceGone = errors.New("rename source was deleted") @@ -527,18 +584,71 @@ func (h *S3TablesHandler) claimForDelete(ctx context.Context, client filer_pb.Se return claim, claimed, nil } -// releaseDeleteClaim removes claim from path while path still holds it, even -// after the request was cancelled. -func (h *S3TablesHandler) releaseDeleteClaim(ctx context.Context, client filer_pb.SeaweedFilerClient, path, claim string) { - err := h.mutateEntryExtended(context.WithoutCancel(ctx), client, path, func(extended map[string][]byte) error { +// tryReleaseDeleteClaim removes claim from path once, while path still holds +// it. A claim another delete replaced or lifted, or an entry that is gone, +// leaves nothing to release and is not an error. +func (h *S3TablesHandler) tryReleaseDeleteClaim(ctx context.Context, client filer_pb.SeaweedFilerClient, path, claim string) error { + err := h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error { if string(extended[ExtendedKeyDeleting]) != claim { return errNoDeleteClaim } delete(extended, ExtendedKeyDeleting) return nil }) - if err != nil && !errors.Is(err, errNoDeleteClaim) && !errors.Is(err, filer_pb.ErrNotFound) { - glog.Warningf("release delete claim on %s: %v", path, err) + if errors.Is(err, errNoDeleteClaim) || errors.Is(err, filer_pb.ErrNotFound) { + return nil + } + return err +} + +// releaseDeleteClaim removes claim from path while path still holds it, even +// after the request was cancelled. A claim left behind refuses creates under a +// bucket or namespace and makes a leftover directory read as taken, so a failed +// release is retried, including one that lost to another write of the entry. +func (h *S3TablesHandler) releaseDeleteClaim(ctx context.Context, client filer_pb.SeaweedFilerClient, path, claim string) { + ctx = context.WithoutCancel(ctx) + var err error + for attempt := 0; attempt < extendedAttributeUpdateAttempts; attempt++ { + if attempt > 0 { + time.Sleep(100 * time.Millisecond) + } + if err = h.tryReleaseDeleteClaim(ctx, client, path, claim); err == nil { + return + } + } + glog.Warningf("release delete claim on %s: %v", path, err) +} + +// isTableBucketCreatedAt accepts only the table bucket created at createdAt. A +// bucket deleted and created again under the same name has a CreatedAt of its +// own, so a create authorized against the first reads the second as gone. +func isTableBucketCreatedAt(createdAt time.Time) func(*filer_pb.Entry) bool { + return func(entry *filer_pb.Entry) bool { + if !IsTableBucketEntry(entry) { + return false + } + var metadata tableBucketMetadata + if json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata) != nil { + return false + } + return metadata.CreatedAt.Equal(createdAt) + } +} + +// isNamespaceCreatedAt is isTableBucketCreatedAt for a namespace its create has +// confirmed: one deleted and created again under the same name reads as the +// authorized one gone. A bucket created again holds only namespaces of its own, +// so a create bound to its namespace this way is bound to the bucket too. +func isNamespaceCreatedAt(createdAt time.Time) func(*filer_pb.Entry) bool { + return func(entry *filer_pb.Entry) bool { + if !isFinishedNamespaceEntry(entry) { + return false + } + var metadata namespaceMetadata + if json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata) != nil { + return false + } + return metadata.CreatedAt.Equal(createdAt) } } @@ -549,7 +659,8 @@ func isNamespaceEntry(entry *filer_pb.Entry) bool { } // checkParentOpen returns errParentGone if parentPath is missing or not a -// catalog entry, and errParentDeleting if a delete has claimed it. +// catalog entry, errParentCreating if the create that wrote it has not +// confirmed it yet, and errParentDeleting if a delete has claimed it. func (h *S3TablesHandler) checkParentOpen(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool) error { _, err := h.openParent(ctx, client, parentPath, isParent) return err @@ -568,6 +679,9 @@ func (h *S3TablesHandler) openParent(ctx context.Context, client filer_pb.Seawee if !isParent(entry) { return nil, errParentGone } + if isUnfinishedCreate(entry.Extended) { + return nil, errParentCreating + } if _, claimed := entry.Extended[ExtendedKeyDeleting]; claimed { return nil, errParentDeleting } @@ -636,6 +750,28 @@ func (h *S3TablesHandler) mutateCatalogRecord( }) } +// requireTableCreatedAt refuses a write to the record at path unless it is the +// table created at createdAt, the one the caller was authorized against: a +// table deleted and created again under the name has a CreatedAt of its own. +// Only the creation time is compared, so the commits an Iceberg writer lands +// meanwhile do not turn a write of the table's policy, tags or maintenance +// settings into a conflict. +func requireTableCreatedAt(extended map[string][]byte, path string, createdAt time.Time) error { + var current tableMetadataInternal + if json.Unmarshal(extended[ExtendedKeyMetadata], ¤t) != nil || !current.CreatedAt.Equal(createdAt) { + return fmt.Errorf("%w: %s holds another table", ErrConcurrentUpdate, path) + } + return nil +} + +// requireTableBucketCreatedAt is requireTableCreatedAt for a table bucket. +func requireTableBucketCreatedAt(extended map[string][]byte, path string, createdAt time.Time) error { + if !isTableBucketCreatedAt(createdAt)(&filer_pb.Entry{Extended: extended}) { + return fmt.Errorf("%w: %s holds another table bucket", ErrConcurrentUpdate, path) + } + return nil +} + // confirmParent re-checks the parent after a create wrote written onto // childPath. If the parent is open, it clears the record's unfinished mark — // then checks the parent once more, since a delete claiming it in between @@ -644,6 +780,20 @@ func (h *S3TablesHandler) mutateCatalogRecord( // it removes what the create wrote and returns why, or errBackOutFailed if // the record stays. func (h *S3TablesHandler) confirmParent(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool, childPath string, written map[string][]byte, emptyDirs ...string) error { + return h.confirmChild(ctx, client, parentPath, isParent, true, childPath, written, emptyDirs) +} + +// confirmParentOnce is confirmParent for a record that other creates build on, +// a namespace. Checking the parent again after the mark comes off would leave a +// window in which a child is created in the record and the check then removes +// the record under it, so the parent is checked once, before the mark comes +// off. That is enough while a delete of the parent counts the marked record: one +// claiming the parent after the check scans the record and is refused. +func (h *S3TablesHandler) confirmParentOnce(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool, childPath string, written map[string][]byte, emptyDirs ...string) error { + return h.confirmChild(ctx, client, parentPath, isParent, false, childPath, written, emptyDirs) +} + +func (h *S3TablesHandler) confirmChild(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool, recheck bool, childPath string, written map[string][]byte, emptyDirs []string) error { parent, err := h.openParent(ctx, client, parentPath, isParent) confirmed := false if err == nil { @@ -651,7 +801,9 @@ func (h *S3TablesHandler) confirmParent(ctx context.Context, client filer_pb.Sea // create wrote; a failed write backs the record out like a closed parent. if err = h.removeExtendedAttributesIf(ctx, client, childPath, written, ExtendedKeyDeleting); err == nil { confirmed = true - err = h.parentStillOpen(ctx, client, parentPath, isParent, parent) + if recheck { + err = h.parentStillOpen(ctx, client, parentPath, isParent, parent) + } } } if err == nil { @@ -753,6 +905,159 @@ func (h *S3TablesHandler) dirHoldsFiles(ctx context.Context, client filer_pb.Sea return false, nil } +// claimTreeDirectory writes a fresh delete claim onto the directory at dirPath +// while it still carries expected, and returns the claim with the attributes it +// left. A claim seen in expected, one a delete that died left, is replaced like +// any other attribute; one written since fails the compare. +// +// The claim is what ties the later delete to this directory: the delete +// requires it, and only this directory carries it. A file or an S3 object at +// dirPath is refused with errDirNotEmpty, whether it was there when the entry +// was read or took the directory's place before the write, which the filer +// refuses since it never writes a directory entry over a file. One that takes +// the place after the write lacks the claim, so the delete is refused instead. +// An S3 object landing on the directory between the read and the write carries +// the prefix-object mark, which the write asserts absent, so the write fails +// instead of putting the bare directory back over the object. +func (h *S3TablesHandler) claimTreeDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, dirPath string, expected map[string][]byte) (string, map[string][]byte, error) { + claim := generateVersionToken() + var claimed map[string][]byte + err := h.mutateEntryAsserting(ctx, client, dirPath, []string{s3_constants.SeaweedFSPrefixObject}, func(entry *filer_pb.Entry) error { + if !entry.IsDirectory || entry.IsDirectoryKeyObject() { + return fmt.Errorf("%w: %s is an object", errDirNotEmpty, dirPath) + } + for key, want := range expected { + if !bytes.Equal(entry.Extended[key], want) { + return fmt.Errorf("%w: %s changed on %s", ErrConcurrentUpdate, key, dirPath) + } + } + entry.Extended[ExtendedKeyDeleting] = []byte(claim) + claimed = snapshotExtended(entry.Extended) + return nil + }) + if errors.Is(err, filer_pb.ErrNotFound) { + return "", nil, fmt.Errorf("%w: %s", ErrConcurrentUpdate, dirPath) + } + if err != nil && !errors.Is(err, errDirNotEmpty) && !errors.Is(err, ErrConcurrentUpdate) { + // The filer reports the refused write over a file only as an opaque + // failure, so look at what holds the path now. + if current, lookupErr := h.lookupEntry(ctx, client, dirPath); lookupErr == nil && !current.IsDirectory { + return "", nil, fmt.Errorf("%w: %s is an object", errDirNotEmpty, dirPath) + } + } + if err != nil { + return "", nil, err + } + return claim, claimed, nil +} + +// removeTreeDirectory removes a directory the walk listed as listed, under a +// claim of its own, and lifts the claim if the delete is refused. +func (h *S3TablesHandler) removeTreeDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, dirPath string, listed *filer_pb.Entry) error { + claim, claimed, err := h.claimTreeDirectory(ctx, client, dirPath, snapshotExtended(listed.Extended)) + if err != nil { + return err + } + if err := h.deleteEmptyDirectoryIfMatches(ctx, client, dirPath, &filer_pb.Entry{Extended: claimed}); err != nil { + h.releaseDeleteClaim(ctx, client, dirPath, claim) + return err + } + return nil +} + +// removeEmptyTree removes dirPath and every directory under it, provided no +// file, no S3 object stored on a directory entry, and no catalog record sits +// anywhere in the tree, dirPath included. dirEntry is dirPath as the caller saw +// it. +// +// dirPath is claimed first, in a write that requires the attributes the caller +// saw: a directory taken back as a namespace since then fails it, before +// anything under it is touched, and the claim keeps it from being taken back +// while the tree goes. With dirPath no namespace, nothing under it can become a +// catalog record either, so a record the walk meets was there before and is +// refused, not removed. +// +// The walk lists the tree a page at a time and stops at the first file, object +// or record with errDirNotEmpty, before anything is removed. Directories then go +// deepest first, each claimed with the attributes it had when listed and +// deleted under that claim, which refuses a directory that holds an entry by +// now or was replaced by a file. dirPath goes last, still under its claim; any +// other way out lifts the claim. +func (h *S3TablesHandler) removeEmptyTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dirPath string, dirEntry *filer_pb.Entry) (err error) { + claim, claimed, err := h.claimTreeDirectory(ctx, client, dirPath, snapshotExtended(dirEntry.Extended)) + if err != nil { + return err + } + defer func() { + if err != nil { + h.releaseDeleteClaim(ctx, client, dirPath, claim) + } + }() + + type listedDir struct { + path string + entry *filer_pb.Entry + } + listed := []listedDir{{dirPath, &filer_pb.Entry{Extended: claimed}}} + + walkCtx, cancel := context.WithCancel(ctx) + defer cancel() + // A directory is listed after its parent, so removing in reverse order + // removes every directory after the ones under it. + for next := 0; next < len(listed); next++ { + dir := listed[next].path + lastFileName := "" + for { + resp, err := client.ListEntries(walkCtx, &filer_pb.ListEntriesRequest{ + Directory: dir, + Limit: dirListPageSize, + StartFromFileName: lastFileName, + InclusiveStartFrom: lastFileName == "", + }) + if err != nil { + if status.Code(err) == codes.NotFound { + break // gone, so its delete below reports the change + } + return err + } + + received := false + for { + entry, err := resp.Recv() + if err != nil { + if err == io.EOF || status.Code(err) == codes.NotFound { + break + } + return err + } + if entry.Entry == nil { + continue + } + received = true + lastFileName = entry.Entry.Name + child := path.Join(dir, entry.Entry.Name) + if !entry.Entry.IsDirectory || entry.Entry.IsDirectoryKeyObject() { + return fmt.Errorf("%w: %s is an object", errDirNotEmpty, child) + } + if _, isRecord := entry.Entry.Extended[ExtendedKeyMetadata]; isRecord { + return fmt.Errorf("%w: %s is a catalog record", errDirNotEmpty, child) + } + listed = append(listed, listedDir{child, entry.Entry}) + } + if !received { + break + } + } + } + + for i := len(listed) - 1; i > 0; i-- { + if err := h.removeTreeDirectory(ctx, client, listed[i].path, listed[i].entry); err != nil { + return err + } + } + return h.deleteEmptyDirectoryIfMatches(ctx, client, dirPath, &filer_pb.Entry{Extended: claimed}) +} + // keepAsLeftover clears a namespace's catalog attributes and the delete's // claim in one write, while the entry still holds claimed, keeping the // directory and the data under it. The write compares against claimed so the diff --git a/weed/s3api/s3tables/handler.go b/weed/s3api/s3tables/handler.go index 2666480e40..8e1b932bac 100644 --- a/weed/s3api/s3tables/handler.go +++ b/weed/s3api/s3tables/handler.go @@ -76,6 +76,7 @@ type S3TablesHandler struct { trusted bool // Trusted local tooling (shell/admin) bypasses authorization iamAuthorizer IAMAuthorizer localAccountIDs func(string) bool + canEncrypt func() bool } // NewS3TablesHandler creates a new S3 Tables handler @@ -119,6 +120,13 @@ func (h *S3TablesHandler) SetLocalAccountChecker(fn func(string) bool) { h.localAccountIDs = fn } +// SetEncryptionChecker injects the predicate that tells whether the gateway +// holds an SSE-S3 key, so a bucket whose declared AES256 the write path +// applies is not created where every write would then fail. +func (h *S3TablesHandler) SetEncryptionChecker(fn func() bool) { + h.canEncrypt = fn +} + // FilerClient interface for filer operations type FilerClient interface { WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error diff --git a/weed/s3api/s3tables/handler_bind_authorized_test.go b/weed/s3api/s3tables/handler_bind_authorized_test.go new file mode 100644 index 0000000000..0544ec5406 --- /dev/null +++ b/weed/s3api/s3tables/handler_bind_authorized_test.go @@ -0,0 +1,610 @@ +package s3tables + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +// recreatedNamespace is the record of a namespace deleted and created again +// under name: another owner, and a creation time of its own. +func recreatedNamespace(t *testing.T, name string) map[string][]byte { + t.Helper() + meta, err := json.Marshal(namespaceMetadata{ + Namespace: []string{name}, + OwnerAccountID: "111122223333", + CreatedAt: time.Now().Add(time.Hour), + }) + require.NoError(t, err) + return map[string][]byte{ExtendedKeyMetadata: meta} +} + +// replaceNamespaceAtLookup replaces namespace with a recreated one at the first +// lookup of namespace/child, the create's look for its own name, which comes +// after the create was authorized against the first namespace. +func replaceNamespaceAtLookup(t *testing.T, fs *s3tablestest.MemFiler, namespace, child string) map[string][]byte { + t.Helper() + bucketPath := GetTableBucketPath(renameTestBucket) + recreated := recreatedNamespace(t, namespace) + fs.OnLookup = func(dir, name string) { + if dir != GetNamespacePath(renameTestBucket, namespace) || name != child { + return + } + fs.OnLookup = nil + fs.Put(bucketPath, namespace, recreated) + } + return recreated +} + +// A create is authorized against the namespace it read. If that namespace is +// deleted and another is created under the same name before the create writes, +// the entry would land in a namespace nobody authorized it for, carrying the +// first one's owner. The create checks the namespace by its creation time, +// reads the new one as the old one gone, and backs out what it wrote. +func TestCreateInNamespaceTreatsARecreatedNamespaceAsGone(t *testing.T) { + for _, tc := range []struct { + operation string + input func(t *testing.T) interface{} + }{ + {"CreateTable", func(t *testing.T) interface{} { return createTableRequest(t, "ns", "t") }}, + {"RegisterTable", func(t *testing.T) interface{} { + return &RegisterTableRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns"}, + Name: "t", + MetadataLocation: "s3://" + renameTestBucket + "/ns/t/metadata/v1.metadata.json", + } + }}, + {"CreateView", func(t *testing.T) interface{} { return createViewRequest(t, "t") }}, + } { + t.Run(tc.operation, func(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + recreated := replaceNamespaceAtLookup(t, fs, "ns", "t") + + code, resp := runHandler(t, m, fs.Client, tc.operation, tc.input(t)) + + assert.Equal(t, http.StatusNotFound, code, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchNamespace, resp.Type, "%+v", resp) + assert.Nil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t"), "nothing is left in the new namespace") + require.NotNil(t, fs.Get(bucketPath, "ns"), "the new namespace stays") + assert.Equal(t, recreated, fs.Get(bucketPath, "ns").Extended, "and is left as it was created") + }) + } +} + +// A rename is authorized to create in the destination namespace it read. If +// that namespace is deleted and created again before the rename publishes, the +// renamed table would land in a namespace nobody authorized it for. The rename +// reads the new one as the old one gone, backs out, and keeps the source. +func TestRenameTableTreatsARecreatedDestinationAsGone(t *testing.T) { + fs, m := startRenameManager(t) + bucketPath := GetTableBucketPath(renameTestBucket) + nsPath := GetNamespacePath(renameTestBucket, "ns") + requireOK(t, m, fs, "CreateNamespace", &CreateNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns2"}, + }) + source := fs.Get(nsPath, "t").Extended + // The rename looks the destination name up right after it read the + // destination namespace. + recreated := replaceNamespaceAtLookup(t, fs, "ns2", "t") + + code, resp := runHandler(t, m, fs.Client, "RenameTable", &RenameTableRequest{ + TableBucketARN: mustBucketARN(t), + SourceNamespace: []string{"ns"}, + SourceName: "t", + DestNamespace: []string{"ns2"}, + DestName: "t", + }) + + assert.Equal(t, http.StatusNotFound, code, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchNamespace, resp.Type, "%+v", resp) + assert.Nil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns2"), "t"), "nothing is left in the new namespace") + require.NotNil(t, fs.Get(bucketPath, "ns2"), "the new namespace stays") + assert.Equal(t, recreated, fs.Get(bucketPath, "ns2").Extended, "and is left as it was created") + require.NotNil(t, fs.Get(nsPath, "t"), "the source stays") + assert.Equal(t, source, fs.Get(nsPath, "t").Extended, "with its record and no claim") +} + +// recreatedRecord is the record of a table or view deleted and created again +// under ns/name: another owner, and a creation time of its own. +func recreatedRecord(t *testing.T, name, entryType string) map[string][]byte { + t.Helper() + now := time.Now().Add(time.Hour) + meta, err := json.Marshal(tableMetadataInternal{ + Name: name, + Namespace: "ns", + Format: FormatIceberg, + CreatedAt: now, + ModifiedAt: now, + OwnerAccountID: "111122223333", + VersionToken: generateVersionToken(), + MetadataVersion: 1, + MetadataLocation: "s3://" + renameTestBucket + "/ns/" + name + "/metadata/v1-recreated.metadata.json", + }) + require.NoError(t, err) + return map[string][]byte{ + ExtendedKeyMetadata: meta, + ExtendedKeyEntryType: []byte(entryType), + } +} + +// startViewV seeds the namespace ns holding the view v. +func startViewV(t *testing.T) (*s3tablestest.MemFiler, *Manager) { + t.Helper() + fs, m := startEmptyBucket(t) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + requireOK(t, m, fs, "CreateView", createViewRequest(t, "v")) + return fs, m +} + +func updateViewRequest(t *testing.T, token, version string) *UpdateViewRequest { + return &UpdateViewRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns"}, + Name: "v", + VersionToken: token, + MetadataLocation: "s3://" + renameTestBucket + "/ns/v/metadata/" + version + ".metadata.json", + } +} + +// An update is authorized against the view it read. If that view is deleted +// and another is created under the same name before the update writes, the +// write would land on a view nobody authorized it for. The write is conditional +// on the record the update read, so it is refused and the new view kept. +func TestUpdateViewSparesARecreatedView(t *testing.T) { + fs, m := startViewV(t) + nsPath := GetNamespacePath(renameTestBucket, "ns") + recreated := recreatedRecord(t, "v", EntryTypeView) + // The update's write is its first; the view is replaced as it lands, so the + // write loses and its retry reads the new view. + fs.BeforeUpdate = func() { fs.Put(nsPath, "v", recreated) } + + code, resp := runHandler(t, m, fs.Client, "UpdateView", updateViewRequest(t, "", "v2")) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + require.NotNil(t, fs.Get(nsPath, "v"), "the new view stays") + assert.Equal(t, recreated, fs.Get(nsPath, "v").Extended, "and is left as it was created") +} + +// Two updates that read the same version both pass the version-token check. +// The second to write must not overwrite the first one's metadata pointer, or +// the version the first published is lost. +func TestUpdateViewRejectsLostUpdate(t *testing.T) { + fs, m := startViewV(t) + token := versionTokenOf(t, fs, "v") + var winnerStatus int + var winnerResp S3TablesError + // The winner runs whole between the loser's read and its write. + fs.BeforeUpdate = func() { + winnerStatus, winnerResp = runHandler(t, m, fs.Client, "UpdateView", updateViewRequest(t, token, "v2-winner")) + } + + code, resp := runHandler(t, m, fs.Client, "UpdateView", updateViewRequest(t, token, "v2-loser")) + + require.Equal(t, http.StatusOK, winnerStatus, "%+v", winnerResp) + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + got := &GetViewResponse{} + require.NoError(t, m.Execute(context.Background(), NewManagerClient(fs.Client), "GetView", getViewRequest(t, "v"), got, "")) + assert.Equal(t, updateViewRequest(t, token, "v2-winner").MetadataLocation, got.MetadataLocation, "the first update must survive") +} + +// A policy written between the authorization check and the write decides +// whether this caller may still update the view, so the update is refused +// rather than applied on a decision that has been overtaken. +func TestUpdateViewRejectsWhenTheAuthorizingPolicyChanges(t *testing.T) { + fs, m := startViewV(t) + nsPath := GetNamespacePath(renameTestBucket, "ns") + policy := []byte(`{"Version":"2012-10-17","Statement":[]}`) + fs.BeforeUpdate = func() { + extended := map[string][]byte{} + for key, value := range fs.Get(nsPath, "v").Extended { + extended[key] = value + } + extended[ExtendedKeyPolicy] = policy + fs.Put(nsPath, "v", extended) + } + + code, resp := runHandler(t, m, fs.Client, "UpdateView", updateViewRequest(t, "", "v2")) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + assert.Equal(t, policy, fs.Get(nsPath, "v").Extended[ExtendedKeyPolicy], "the concurrent policy write must survive") +} + +const bindTestPolicy = `{"Statement":[{"Effect":"Allow","Principal":"writer","Action":"s3tables:*","Resource":"*"}]}` + +// tableWriteOp is a write to the table ns/t, with the attributes the table +// carries for it besides its record. +type tableWriteOp struct { + operation string + input func(t *testing.T) interface{} + extra map[string][]byte +} + +func tableWriteOps() []tableWriteOp { + tableARN := func(t *testing.T) string { + arn, err := BuildTableARN(DefaultRegion, DefaultAccountID, renameTestBucket, "ns", "t") + require.NoError(t, err) + return arn + } + return []tableWriteOp{ + {operation: "PutTablePolicy", input: func(t *testing.T) interface{} { + return &PutTablePolicyRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ns"}, Name: "t", ResourcePolicy: bindTestPolicy} + }}, + {operation: "DeleteTablePolicy", input: func(t *testing.T) interface{} { + return &DeleteTablePolicyRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ns"}, Name: "t"} + }, extra: map[string][]byte{ExtendedKeyPolicy: []byte(bindTestPolicy)}}, + {operation: "TagResource", input: func(t *testing.T) interface{} { + return &TagResourceRequest{ResourceARN: tableARN(t), Tags: map[string]string{"k": "v"}} + }}, + {operation: "UntagResource", input: func(t *testing.T) interface{} { + return &UntagResourceRequest{ResourceARN: tableARN(t), TagKeys: []string{"k"}} + }, extra: map[string][]byte{ExtendedKeyTags: []byte(`{"k":"v","keep":"1"}`)}}, + {operation: "PutTableMaintenanceConfiguration", input: func(t *testing.T) interface{} { + return &PutTableMaintenanceConfigurationRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns"}, + Name: "t", + Type: MaintenanceTypeIcebergCompaction, + Value: &MaintenanceConfigurationValue{Status: MaintenanceStatusEnabled}, + } + }}, + } +} + +// startTableT seeds the namespace ns holding the table t, carrying extra +// besides its record. +func startTableT(t *testing.T, extra map[string][]byte) (*s3tablestest.MemFiler, *Manager) { + t.Helper() + fs, m := startRenameManager(t) + nsPath := GetNamespacePath(renameTestBucket, "ns") + extended := extendedOf(t, fs, nsPath, "t") + for key, value := range extra { + extended[key] = value + } + fs.Put(nsPath, "t", extended) + return fs, m +} + +// A write to a table's policy, tags or maintenance settings is authorized +// against the table it read. If that table is deleted and another is created +// under the same name before the write, it would land on a table nobody +// authorized it for: the policy or tag values it compares can match the new +// table's just as well. The write checks the table by its creation time and is +// refused, and the new table is kept as it was created. +func TestTableWritesSpareARecreatedTable(t *testing.T) { + for _, op := range tableWriteOps() { + t.Run(op.operation, func(t *testing.T) { + fs, m := startTableT(t, op.extra) + nsPath := GetNamespacePath(renameTestBucket, "ns") + recreated := recreatedRecord(t, "t", EntryTypeTable) + for key, value := range op.extra { + recreated[key] = value + } + // The write is the operation's first; the table is replaced as it + // lands, so the write loses and its retry reads the new table. + fs.BeforeUpdate = func() { fs.Put(nsPath, "t", recreated) } + + code, resp := runHandler(t, m, fs.Client, op.operation, op.input(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + require.NotNil(t, fs.Get(nsPath, "t"), "the new table stays") + assert.Equal(t, recreated, fs.Get(nsPath, "t").Extended, "and is left as it was created") + }) + } +} + +// The same writes compare only the table's creation time, not its metadata: +// an Iceberg commit to the table between the read and the write is the same +// table, and must not turn the write into a conflict. +func TestTableWritesTolerateACommitInBetween(t *testing.T) { + for _, op := range tableWriteOps() { + t.Run(op.operation, func(t *testing.T) { + fs, m := startTableT(t, op.extra) + nsPath := GetNamespacePath(renameTestBucket, "ns") + committedLocation := "s3://" + renameTestBucket + "/ns/t/metadata/v4.metadata.json" + fs.BeforeUpdate = func() { + extended := extendedOf(t, fs, nsPath, "t") + var metadata tableMetadataInternal + require.NoError(t, json.Unmarshal(extended[ExtendedKeyMetadata], &metadata)) + metadata.MetadataVersion = 4 + metadata.MetadataLocation = committedLocation + metadata.VersionToken = generateVersionToken() + metadata.ModifiedAt = time.Now() + data, err := json.Marshal(metadata) + require.NoError(t, err) + extended[ExtendedKeyMetadata] = data + fs.Put(nsPath, "t", extended) + } + + requireOK(t, m, fs, op.operation, op.input(t)) + + got, err := runGetTable(t, m, fs, "ns", "t") + require.NoError(t, err) + assert.Equal(t, committedLocation, got.MetadataLocation, "the commit stays") + }) + } +} + +// namespaceMetadataOf reads the record of a namespace in the test bucket. +func namespaceMetadataOf(t *testing.T, fs *s3tablestest.MemFiler, namespace string) namespaceMetadata { + t.Helper() + entry := fs.Get(GetTableBucketPath(renameTestBucket), namespace) + require.NotNil(t, entry) + var metadata namespaceMetadata + require.NoError(t, json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata)) + return metadata +} + +// updateNamespaceAtLookup runs a real UpdateNamespace the first time child is +// looked up under namespace: after the request was authorized against the +// namespace and before it writes. The answer fills in once the update has run. +func updateNamespaceAtLookup(t *testing.T, fs *s3tablestest.MemFiler, m *Manager, namespace, child string) *handlerResult { + t.Helper() + update := &handlerResult{} + fs.OnLookup = func(dir, name string) { + if dir != GetNamespacePath(renameTestBucket, namespace) || name != child { + return + } + fs.OnLookup = nil + update.status, update.resp = runHandler(t, m, fs.Client, "UpdateNamespace", &UpdateNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{namespace}, + Properties: map[string]string{"edited": "true"}, + }) + } + return update +} + +// requireNamespaceUpdatedInPlace checks that the update ran, took effect, and +// kept the namespace the record it was: same owner, same creation time. +func requireNamespaceUpdatedInPlace(t *testing.T, fs *s3tablestest.MemFiler, namespace string, before namespaceMetadata, update *handlerResult) { + t.Helper() + require.NotZero(t, update.status, "the update has to land between the authorization and the write") + require.Equal(t, http.StatusOK, update.status, "%+v", update.resp) + after := namespaceMetadataOf(t, fs, namespace) + assert.Equal(t, map[string]string{"edited": "true"}, after.Properties) + assert.True(t, after.CreatedAt.Equal(before.CreatedAt), "an update must keep the creation time: %v became %v", before.CreatedAt, after.CreatedAt) + assert.Equal(t, before.OwnerAccountID, after.OwnerAccountID) +} + +// A namespace update edits the record a request was authorized against; it +// does not replace it. The creation time survives, so the request still writes: +// reading the edit as a delete and create would back the entry out and answer +// 404 for a namespace that never went away. Unlike +// TestCreateConfirmingAcrossANamespaceUpdateSucceeds, the update here is the +// real UpdateNamespace and lands before the first check against the authorized +// record, not after it. +func TestCreateAuthorizedBeforeANamespaceUpdateSucceeds(t *testing.T) { + for _, op := range createOps(t) { + t.Run(op.operation, func(t *testing.T) { + fs, m := startEmptyBucket(t) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + before := namespaceMetadataOf(t, fs, "ns") + update := updateNamespaceAtLookup(t, fs, m, "ns", lateEntry) + + code, resp := runHandler(t, m, fs.Client, op.operation, op.input) + + assert.Equal(t, http.StatusOK, code, "%+v", resp) + requireNamespaceUpdatedInPlace(t, fs, "ns", before, update) + entry := fs.Get(GetNamespacePath(renameTestBucket, "ns"), lateEntry) + require.NotNil(t, entry, "the entry the request answered for must exist") + assert.Contains(t, entry.Extended, ExtendedKeyMetadata) + assert.NotContains(t, entry.Extended, ExtendedKeyDeleting, "and carry no unfinished mark") + }) + } + + // A rename is bound to its destination namespace the same way. + for _, tc := range []struct { + operation string + name string + seed func(t *testing.T, fs *s3tablestest.MemFiler, name string) + }{ + {"RenameTable", "t", nil}, + {"RenameView", "v", seedView}, + } { + t.Run(tc.operation, func(t *testing.T) { + fs, m := startRenameManager(t) + if tc.seed != nil { + tc.seed(t, fs, tc.name) + } + requireOK(t, m, fs, "CreateNamespace", &CreateNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns2"}, + }) + before := namespaceMetadataOf(t, fs, "ns2") + update := updateNamespaceAtLookup(t, fs, m, "ns2", tc.name) + + code, resp := runHandler(t, m, fs.Client, tc.operation, &RenameTableRequest{ + TableBucketARN: mustBucketARN(t), + SourceNamespace: []string{"ns"}, + SourceName: tc.name, + DestNamespace: []string{"ns2"}, + DestName: tc.name, + }) + + assert.Equal(t, http.StatusOK, code, "%+v", resp) + requireNamespaceUpdatedInPlace(t, fs, "ns2", before, update) + dest := fs.Get(GetNamespacePath(renameTestBucket, "ns2"), tc.name) + require.NotNil(t, dest, "the renamed entry must exist") + assert.Contains(t, dest.Extended, ExtendedKeyMetadata) + assert.NotContains(t, dest.Extended, ExtendedKeyDeleting, "and carry no unfinished mark") + }) + } +} + +// writeRecorder remembers the writes that reach a directory tree, so a test can +// tell a request that never wrote there from one that wrote and backed out. +type writeRecorder struct { + filer_pb.SeaweedFilerClient + tree string + mu sync.Mutex + writes []string +} + +func (c *writeRecorder) note(op, dir, name string) { + if dir != c.tree && !strings.HasPrefix(dir, c.tree+"/") { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.writes = append(c.writes, op+" "+dir+"/"+name) +} + +func (c *writeRecorder) CreateEntry(ctx context.Context, in *filer_pb.CreateEntryRequest, opts ...grpc.CallOption) (*filer_pb.CreateEntryResponse, error) { + c.note("create", in.Directory, in.Entry.GetName()) + return c.SeaweedFilerClient.CreateEntry(ctx, in, opts...) +} + +func (c *writeRecorder) UpdateEntry(ctx context.Context, in *filer_pb.UpdateEntryRequest, opts ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) { + c.note("update", in.Directory, in.Entry.GetName()) + return c.SeaweedFilerClient.UpdateEntry(ctx, in, opts...) +} + +func (c *writeRecorder) DeleteEntry(ctx context.Context, in *filer_pb.DeleteEntryRequest, opts ...grpc.CallOption) (*filer_pb.DeleteEntryResponse, error) { + c.note("delete", in.Directory, in.Name) + return c.SeaweedFilerClient.DeleteEntry(ctx, in, opts...) +} + +// A namespace deleted and created again between the authorization and the +// early check is not the one the caller was authorized against, so the request +// is refused before it writes anything: the check after the write only backs +// the record out, and a back-out the filer turns away would leave it in a +// namespace nobody authorized it for. +func TestCreateRefusedBeforeWritingIntoAReplacedNamespace(t *testing.T) { + for _, op := range createOps(t) { + t.Run(op.operation, func(t *testing.T) { + fs, m := startEmptyBucket(t) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + bucketPath := GetTableBucketPath(renameTestBucket) + recreated := recreatedNamespace(t, "ns") + // The create reads the namespace record three times before it writes: for + // its metadata, for its policy, and for the early check. The namespace is + // replaced as the third read starts. + const earlyCheck = 3 + reads := 0 + fs.OnLookup = func(dir, name string) { + if dir != bucketPath || name != "ns" { + return + } + if reads++; reads == earlyCheck { + fs.OnLookup = nil + fs.Put(bucketPath, "ns", recreated) + } + } + recorder := &writeRecorder{SeaweedFilerClient: fs.Client, tree: GetNamespacePath(renameTestBucket, "ns")} + + code, resp := runHandler(t, m, recorder, op.operation, op.input) + + require.GreaterOrEqual(t, reads, earlyCheck, "the namespace has to be replaced before the early check") + assert.Equal(t, http.StatusNotFound, code, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchNamespace, resp.Type, "%+v", resp) + assert.Empty(t, recorder.writes, "nothing is written into the namespace that replaced the authorized one") + assert.Equal(t, recreated, fs.Get(bucketPath, "ns").Extended, "and it is left as it was created") + }) + } + + // A rename reads its destination namespace and then the destination name, + // both before the early check. + for _, tc := range []struct { + operation string + name string + seed func(t *testing.T, fs *s3tablestest.MemFiler, name string) + }{ + {"RenameTable", "t", nil}, + {"RenameView", "v", seedView}, + } { + t.Run(tc.operation, func(t *testing.T) { + fs, m := startRenameManager(t) + if tc.seed != nil { + tc.seed(t, fs, tc.name) + } + requireOK(t, m, fs, "CreateNamespace", &CreateNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns2"}, + }) + recreated := replaceNamespaceAtLookup(t, fs, "ns2", tc.name) + recorder := &writeRecorder{SeaweedFilerClient: fs.Client, tree: GetNamespacePath(renameTestBucket, "ns2")} + + code, resp := runHandler(t, m, recorder, tc.operation, &RenameTableRequest{ + TableBucketARN: mustBucketARN(t), + SourceNamespace: []string{"ns"}, + SourceName: tc.name, + DestNamespace: []string{"ns2"}, + DestName: tc.name, + }) + + assert.Equal(t, http.StatusNotFound, code, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchNamespace, resp.Type, "%+v", resp) + assert.Empty(t, recorder.writes, "nothing is written into the namespace that replaced the authorized one") + assert.Equal(t, recreated, fs.Get(GetTableBucketPath(renameTestBucket), "ns2").Extended, "and it is left as it was created") + }) + } +} + +// UpdateNamespace writes the record it was authorized against, owner and +// creation time included. A namespace deleted and created again before that +// write is refused, not handed the first one's owner. +func TestUpdateNamespaceSparesARecreatedNamespace(t *testing.T) { + fs, m := startEmptyBucket(t) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + bucketPath := GetTableBucketPath(renameTestBucket) + recreated := recreatedNamespace(t, "ns") + fs.OnUpdate = func(dir, name string) { + if dir != bucketPath || name != "ns" { + return + } + fs.OnUpdate = nil + fs.Put(bucketPath, "ns", recreated) + } + + code, resp := runHandler(t, m, fs.Client, "UpdateNamespace", &UpdateNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns"}, + Properties: map[string]string{"edited": "true"}, + }) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, recreated, fs.Get(bucketPath, "ns").Extended, "the new namespace is untouched") +} + +// DeleteNamespace removes only the namespace it was authorized against: one +// deleted and created again before the claim reads as the first one gone. +func TestDeleteNamespaceSparesARecreatedNamespace(t *testing.T) { + fs, m := startEmptyBucket(t) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + bucketPath := GetTableBucketPath(renameTestBucket) + recreated := recreatedNamespace(t, "ns") + fs.OnUpdate = func(dir, name string) { + if dir != bucketPath || name != "ns" { + return + } + fs.OnUpdate = nil + fs.Put(bucketPath, "ns", recreated) + } + + code, resp := runHandler(t, m, fs.Client, "DeleteNamespace", &DeleteNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns"}, + }) + + assert.Equal(t, http.StatusNotFound, code, "%+v", resp) + require.NotNil(t, fs.Get(bucketPath, "ns"), "the new namespace survives") + assert.Equal(t, recreated, fs.Get(bucketPath, "ns").Extended, "the new namespace is untouched") +} diff --git a/weed/s3api/s3tables/handler_bind_bucket_test.go b/weed/s3api/s3tables/handler_bind_bucket_test.go new file mode 100644 index 0000000000..e36229a897 --- /dev/null +++ b/weed/s3api/s3tables/handler_bind_bucket_test.go @@ -0,0 +1,104 @@ +package s3tables + +import ( + "encoding/json" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// recreatedBucket is the record of the test bucket deleted and created again: +// another owner, and a creation time of its own. +func recreatedBucket(t *testing.T) map[string][]byte { + t.Helper() + meta, err := json.Marshal(tableBucketMetadata{ + Name: renameTestBucket, + OwnerAccountID: "111122223333", + CreatedAt: time.Now().Add(time.Hour), + }) + require.NoError(t, err) + return map[string][]byte{ + ExtendedKeyTableBucket: []byte("{}"), + ExtendedKeyMetadata: meta, + } +} + +// bucketWriteOp is a write to the test bucket, with the attributes the bucket +// carries for it besides its record. +type bucketWriteOp struct { + operation string + input func(t *testing.T) interface{} + extra map[string][]byte +} + +func bucketWriteOps() []bucketWriteOp { + return []bucketWriteOp{ + {operation: "PutTableBucketPolicy", input: func(t *testing.T) interface{} { + return &PutTableBucketPolicyRequest{TableBucketARN: mustBucketARN(t), ResourcePolicy: bindTestPolicy} + }}, + {operation: "DeleteTableBucketPolicy", input: func(t *testing.T) interface{} { + return &DeleteTableBucketPolicyRequest{TableBucketARN: mustBucketARN(t)} + }, extra: map[string][]byte{ExtendedKeyPolicy: []byte(bindTestPolicy)}}, + {operation: "TagResource", input: func(t *testing.T) interface{} { + return &TagResourceRequest{ResourceARN: mustBucketARN(t), Tags: map[string]string{"k": "v"}} + }}, + {operation: "UntagResource", input: func(t *testing.T) interface{} { + return &UntagResourceRequest{ResourceARN: mustBucketARN(t), TagKeys: []string{"k"}} + }, extra: map[string][]byte{ExtendedKeyTags: []byte(`{"k":"v","keep":"1"}`)}}, + {operation: "PutTableBucketMaintenanceConfiguration", input: func(t *testing.T) interface{} { + return &PutTableBucketMaintenanceConfigurationRequest{ + TableBucketARN: mustBucketARN(t), + Type: MaintenanceTypeIcebergUnreferencedFileRemoval, + Value: &MaintenanceConfigurationValue{Status: MaintenanceStatusEnabled}, + } + }}, + } +} + +// A write to a bucket's policy, tags or maintenance settings is authorized +// against the bucket it read. If that bucket is deleted and another is created under the same name +// before the write, the values it compares can match the new bucket's just as +// well. The write checks the bucket by its creation time and is refused, and +// the new bucket is kept as it was created. +func TestBucketWritesSpareARecreatedBucket(t *testing.T) { + for _, op := range bucketWriteOps() { + t.Run(op.operation, func(t *testing.T) { + fs, m := startEmptyBucket(t) + extended := extendedOf(t, fs, TablesPath, renameTestBucket) + recreated := recreatedBucket(t) + for key, value := range op.extra { + extended[key] = value + recreated[key] = value + } + fs.Put(TablesPath, renameTestBucket, extended) + fs.BeforeUpdate = func() { fs.Put(TablesPath, renameTestBucket, recreated) } + + code, resp := runHandler(t, m, fs.Client, op.operation, op.input(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + assert.Equal(t, recreated, fs.Get(TablesPath, renameTestBucket).Extended, "the new bucket is left as it was created") + }) + } +} + +// The same writes still land on the bucket they were authorized against. +func TestBucketWritesLandOnTheAuthorizedBucket(t *testing.T) { + for _, op := range bucketWriteOps() { + t.Run(op.operation, func(t *testing.T) { + fs, m := startEmptyBucket(t) + extended := extendedOf(t, fs, TablesPath, renameTestBucket) + for key, value := range op.extra { + extended[key] = value + } + fs.Put(TablesPath, renameTestBucket, extended) + + code, resp := runHandler(t, m, fs.Client, op.operation, op.input(t)) + + assert.Equal(t, http.StatusOK, code, "%+v", resp) + }) + } +} diff --git a/weed/s3api/s3tables/handler_bind_tags_test.go b/weed/s3api/s3tables/handler_bind_tags_test.go new file mode 100644 index 0000000000..cce363229e --- /dev/null +++ b/weed/s3api/s3tables/handler_bind_tags_test.go @@ -0,0 +1,96 @@ +package s3tables + +import ( + "context" + "encoding/json" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Tag operations bound to a table bucket answer not found once its name +// holds another bucket, for the bucket and for a table in it, and leave the +// tags as they were. +func TestTagOperationsKeepToTheBoundBucket(t *testing.T) { + fs, m := startRenameManager(t) + client := NewManagerClient(fs.Client) + tableARN, err := BuildTableARN(DefaultRegion, DefaultAccountID, renameTestBucket, "ns", "t") + require.NoError(t, err) + var bucket GetTableBucketResponse + require.NoError(t, m.Execute(context.Background(), client, "GetTableBucket", + &GetTableBucketRequest{TableBucketARN: mustBucketARN(t)}, &bucket, "")) + bound := WithTableBucketCreatedAt(context.Background(), bucket.CreatedAt) + stale := WithTableBucketCreatedAt(context.Background(), bucket.CreatedAt.Add(-time.Hour)) + + for _, arn := range []string{mustBucketARN(t), tableARN} { + require.NoError(t, m.Execute(bound, client, "TagResource", + &TagResourceRequest{ResourceARN: arn, Tags: map[string]string{"k": "v"}}, nil, ""), arn) + for operation, req := range map[string]interface{}{ + "TagResource": &TagResourceRequest{ResourceARN: arn, Tags: map[string]string{"k": "other"}}, + "UntagResource": &UntagResourceRequest{ResourceARN: arn, TagKeys: []string{"k"}}, + "ListTagsForResource": &ListTagsForResourceRequest{ResourceARN: arn}, + } { + err := m.Execute(stale, client, operation, req, nil, "") + var apiErr *S3TablesError + require.ErrorAs(t, err, &apiErr, "%s on %s", operation, arn) + assert.Contains(t, []string{ErrCodeNoSuchBucket, ErrCodeNoSuchTable}, apiErr.Type, "%s on %s", operation, arn) + } + var tags ListTagsForResourceResponse + require.NoError(t, m.Execute(bound, client, "ListTagsForResource", + &ListTagsForResourceRequest{ResourceARN: arn}, &tags, ""), arn) + assert.Equal(t, map[string]string{"k": "v"}, tags.Tags, arn) + } +} + +// A tag write merges into the tags the record holds when it is written, so a +// tag write that lands between its read and its write is kept. +func TestTagWritesKeepAConcurrentTagWrite(t *testing.T) { + for _, tc := range []struct { + operation string + input func(t *testing.T) interface{} + want map[string]string + }{ + {"TagResource", func(t *testing.T) interface{} { + return &TagResourceRequest{ResourceARN: mustBucketARN(t), Tags: map[string]string{"k": "v"}} + }, map[string]string{"keep": "1", "other": "1", "k": "v"}}, + {"UntagResource", func(t *testing.T) interface{} { + return &UntagResourceRequest{ResourceARN: mustBucketARN(t), TagKeys: []string{"keep"}} + }, map[string]string{"other": "1"}}, + } { + t.Run(tc.operation, func(t *testing.T) { + fs, m := startEmptyBucket(t) + extended := extendedOf(t, fs, TablesPath, renameTestBucket) + extended[ExtendedKeyTags] = []byte(`{"keep":"1"}`) + fs.Put(TablesPath, renameTestBucket, extended) + fs.BeforeUpdate = func() { + current := extendedOf(t, fs, TablesPath, renameTestBucket) + current[ExtendedKeyTags] = []byte(`{"keep":"1","other":"1"}`) + fs.Put(TablesPath, renameTestBucket, current) + } + code, resp := runHandler(t, m, fs.Client, tc.operation, tc.input(t)) + require.Equal(t, http.StatusOK, code, resp.Message) + var got map[string]string + require.NoError(t, json.Unmarshal(extendedOf(t, fs, TablesPath, renameTestBucket)[ExtendedKeyTags], &got)) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestTagResourceKeepsToItsLimit(t *testing.T) { + fs, m := startEmptyBucket(t) + client := NewManagerClient(fs.Client) + ctx := WithMaxTags(context.Background(), 2) + tag := func(tags map[string]string) error { + return m.Execute(ctx, client, "TagResource", &TagResourceRequest{ResourceARN: mustBucketARN(t), Tags: tags}, nil, "") + } + require.NoError(t, tag(map[string]string{"a": "1", "b": "1"})) + require.NoError(t, tag(map[string]string{"a": "2"})) + err := tag(map[string]string{"c": "1"}) + var apiErr *S3TablesError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, ErrCodeInvalidRequest, apiErr.Type) + assert.JSONEq(t, `{"a":"2","b":"1"}`, string(extendedOf(t, fs, TablesPath, renameTestBucket)[ExtendedKeyTags])) +} diff --git a/weed/s3api/s3tables/handler_bucket_create.go b/weed/s3api/s3tables/handler_bucket_create.go index 5d6464c0e1..0370354798 100644 --- a/weed/s3api/s3tables/handler_bucket_create.go +++ b/weed/s3api/s3tables/handler_bucket_create.go @@ -80,6 +80,12 @@ func (h *S3TablesHandler) handleCreateTableBucket(w http.ResponseWriter, r *http } } + if len(req.EncryptionConfiguration) > 0 && IsCatalogOnlyFormat(bucketFormat) && h.canEncrypt != nil && !h.canEncrypt() { + msg := "AES256 encryption needs an SSE-S3 key on the S3 gateway: set s3.sse.kek or s3.sse.key in security.toml" + h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, msg) + return errors.New(msg) + } + bucketPath := GetTableBucketPath(req.Name) // Check if bucket already exists and ensure no conflict with object store buckets diff --git a/weed/s3api/s3tables/handler_bucket_format_test.go b/weed/s3api/s3tables/handler_bucket_format_test.go index b024314a76..7787b8c913 100644 --- a/weed/s3api/s3tables/handler_bucket_format_test.go +++ b/weed/s3api/s3tables/handler_bucket_format_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "testing" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" "github.com/stretchr/testify/require" @@ -107,3 +109,39 @@ func TestNormalizeFormat(t *testing.T) { } } } + +func TestDeclaredBucketEncryption(t *testing.T) { + entry := func(format string, cfg map[string]interface{}) *filer_pb.Entry { + meta, _ := json.Marshal(tableBucketMetadata{Name: "b", Format: format, EncryptionConfiguration: cfg}) + return &filer_pb.Entry{Extended: map[string][]byte{ExtendedKeyTableBucket: []byte("true"), ExtendedKeyMetadata: meta}} + } + aes := map[string]interface{}{"sseAlgorithm": "AES256"} + + require.Equal(t, "AES256", DeclaredBucketEncryption(entry(FormatLance, aes))) + require.Empty(t, DeclaredBucketEncryption(entry(FormatLance, nil))) + require.Empty(t, DeclaredBucketEncryption(entry(FormatIceberg, aes)), "the Iceberg catalog reads client manifests raw") + require.Empty(t, DeclaredBucketEncryption(entry("", aes)), "a bucket without a format is Iceberg") + require.Empty(t, DeclaredBucketEncryption(&filer_pb.Entry{Extended: map[string][]byte{ExtendedKeyMetadata: entry(FormatLance, aes).Extended[ExtendedKeyMetadata]}}), "not a table bucket") + require.Empty(t, DeclaredBucketEncryption(nil)) +} + +// Only a format whose files the catalog leaves to clients gets its declared +// AES256 applied, so only that format needs the gateway's SSE-S3 key. +func TestCreateTableBucketNeedsSSES3ForADeclaredLanceBucket(t *testing.T) { + fs := s3tablestest.Start(t) + m := NewManager() + m.SetEncryptionChecker(func() bool { return false }) + create := func(name, format string, cfg map[string]interface{}) error { + return m.Execute(context.Background(), NewManagerClient(fs.Client), "CreateTableBucket", + &CreateTableBucketRequest{Name: name, Format: format, EncryptionConfiguration: cfg}, nil, s3_constants.AccountAdminId) + } + aes := map[string]interface{}{"sseAlgorithm": "AES256"} + + err := create("lance-sealed", FormatLance, aes) + var s3Err *S3TablesError + require.ErrorAs(t, err, &s3Err) + require.Equal(t, ErrCodeInvalidRequest, s3Err.Type) + require.Contains(t, s3Err.Message, "s3.sse.kek") + require.NoError(t, create("lance-plain", FormatLance, nil)) + require.NoError(t, create("iceberg-sealed", FormatIceberg, aes), "an Iceberg bucket only records the declaration") +} diff --git a/weed/s3api/s3tables/handler_bucket_get_list_delete.go b/weed/s3api/s3tables/handler_bucket_get_list_delete.go index 1b801eb84a..34b8b8ab04 100644 --- a/weed/s3api/s3tables/handler_bucket_get_list_delete.go +++ b/weed/s3api/s3tables/handler_bucket_get_list_delete.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "path" "strings" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -275,6 +276,32 @@ func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http var metadata tableBucketMetadata var bucketPolicy string hasChildren := false + heldDir := "" + unfinishedNamespace := "" + // What the scan found besides namespaces: the bare directories — a deleted + // namespace's leftovers or hidden ones — as they were seen, and the names + // of the hidden files. + var leftovers []*filer_pb.Entry + var hidden []string + // The claim this delete holds on the bucket. It goes with the bucket when it + // is deleted; any other way out lifts it, or the bucket would refuse every + // create. bucketClaimed is the bucket's record as the claim left it, which the + // removal has to find. + bucketClaim := "" + var bucketClaimed map[string][]byte + defer func() { + if bucketClaim == "" { + return + } + // A claim left behind refuses every create in the bucket until the next + // delete replaces it, so a failed release is retried. + if err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + h.releaseDeleteClaim(r.Context(), client, bucketPath, bucketClaim) + return nil + }); err != nil { + glog.Warningf("release delete claim on %s: %v", bucketPath, err) + } + }() err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { // 1. Get metadata for ownership check data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata) @@ -284,6 +311,9 @@ func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http if err := json.Unmarshal(data, &metadata); err != nil { return fmt.Errorf("failed to unmarshal metadata: %w", err) } + if otherTableBucket(r.Context(), metadata.CreatedAt) { + return filer_pb.ErrNotFound + } // Fetch bucket policy if it exists policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy) @@ -310,25 +340,89 @@ func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http return NewAuthError("DeleteTableBucket", principal, fmt.Sprintf("not authorized to delete bucket %s", bucketName)) } - // 3. Check if bucket is empty - resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{ - Directory: bucketPath, - Limit: 10, - }) + // 2. Claim the bucket before scanning it. CreateNamespace refuses a + // claimed bucket and checks it again after its write, so a namespace is + // either seen by the scan below or backed out. The bucket is removed only + // while it still carries this claim: once another delete has replaced or + // lifted it, a create may have passed that check, and this delete answers + // 409 instead. A claim an earlier delete left behind is replaced. Only the + // bucket authorized above is claimed: one deleted and created again under + // the same name since then is another bucket and reads as gone. + bucketClaim, bucketClaimed, err = h.claimForDelete(r.Context(), client, bucketPath, isTableBucketCreatedAt(metadata.CreatedAt)) if err != nil { return err } - for { - entry, err := resp.Recv() + // 3. Check if bucket is empty. A file, an S3 object stored on a directory + // entry, or a namespace is a child. A bare directory is what a deleted + // namespace leaves behind to keep the data under it, so it is only a + // child while it still holds a file. The scan + // pages through the whole bucket: the first page may be nothing but bare + // directories sorted ahead of the namespaces. + lastFileName := "" + for !hasChildren { + resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{ + Directory: bucketPath, + Limit: 10, + StartFromFileName: lastFileName, + InclusiveStartFrom: lastFileName == "", + }) if err != nil { - if err == io.EOF { - break - } return err } - if entry.Entry != nil && !strings.HasPrefix(entry.Entry.Name, ".") { - hasChildren = true + + hasMore := false + for { + entry, err := resp.Recv() + if err != nil { + if err == io.EOF { + break + } + return err + } + if entry.Entry == nil { + continue + } + hasMore = true + lastFileName = entry.Entry.Name + // Hidden entries are never children. A hidden directory can + // hold files the same way a leftover does, so it waits with + // them for the walk; a hidden file is removed by name. + if strings.HasPrefix(entry.Entry.Name, ".") { + if entry.Entry.IsDirectory { + leftovers = append(leftovers, entry.Entry) + } else { + hidden = append(hidden, entry.Entry.Name) + } + continue + } + if _, hasMetadata := entry.Entry.Extended[ExtendedKeyMetadata]; hasMetadata || !entry.Entry.IsDirectory || entry.Entry.IsDirectoryKeyObject() { + hasChildren = true + if hasMetadata && isUnfinishedCreate(entry.Entry.Extended) { + unfinishedNamespace = entry.Entry.Name + } + break + } + leftovers = append(leftovers, entry.Entry) + } + if !hasMore { + break + } + } + + // Looked at after the scan, not during it, so a bucket that still has a + // namespace answers at once instead of walking the leftovers first. The + // first directory holding a file settles it. + if hasChildren { + return nil + } + for _, leftover := range leftovers { + holdsFiles, err := h.dirHoldsFiles(r.Context(), client, path.Join(bucketPath, leftover.Name)) + if err != nil { + return err + } + if holdsFiles { + heldDir = leftover.Name break } } @@ -341,6 +435,8 @@ func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) } else if isAuthError(err) { h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) + } else if errors.Is(err, ErrConcurrentUpdate) { + h.writeError(w, http.StatusConflict, ErrCodeConflict, "table bucket changed during delete, retry the request") } else { h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to delete table bucket: %v", err)) } @@ -348,25 +444,59 @@ func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http } if hasChildren { - h.writeError(w, http.StatusConflict, ErrCodeBucketNotEmpty, "table bucket is not empty") + message := "table bucket is not empty" + if unfinishedNamespace != "" { + // Listings hide it, so name it: creating it again finishes it. + message = fmt.Sprintf("table bucket is not empty: namespace %s has an unfinished create", unfinishedNamespace) + } + h.writeError(w, http.StatusConflict, ErrCodeBucketNotEmpty, message) return fmt.Errorf("bucket not empty") } + // Nothing in the bucket is a namespace, but data sits under a directory a + // deleted namespace left. Deleting the bucket would take it, so it stays. + if heldDir != "" { + h.writeError(w, http.StatusConflict, ErrCodeBucketNotEmpty, fmt.Sprintf("table bucket is not empty: directory %s still holds files", heldDir)) + return fmt.Errorf("bucket not empty: directory %s still holds files", heldDir) + } + // Delete the bucket err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { // Drop the leaf entry first, so it does not outlive the directory. if tableObjErr := h.deleteEntryIfExists(r.Context(), client, GetTableObjectBucketPath(bucketName)); tableObjErr != nil { glog.V(1).Infof("failed to delete table object for %s: %v", bucketName, tableObjErr) } + // Only the hidden files go by a recursive delete. A file written since + // the scan anywhere else makes the filer refuse the delete that would have + // taken it: each leftover goes only if it is still the bare directory the + // scan saw and holds no file or catalog record, and the bucket only if + // nothing is left in it. + for _, leftover := range leftovers { + if err := h.removeEmptyTree(r.Context(), client, path.Join(bucketPath, leftover.Name), leftover); err != nil { + return err + } + } + for _, name := range hidden { + if err := h.deleteDirectory(r.Context(), client, path.Join(bucketPath, name)); err != nil { + return err + } + } // The bucket is the directory, so a refused delete leaves it in place // and the caller must hear about it. - return h.deleteDirectory(r.Context(), client, bucketPath) + return h.deleteEmptyDirectoryIfMatches(r.Context(), client, bucketPath, &filer_pb.Entry{Extended: bucketClaimed}) }) if err != nil { - h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table bucket") + if errors.Is(err, errDirNotEmpty) { + h.writeError(w, http.StatusConflict, ErrCodeBucketNotEmpty, "table bucket is not empty") + } else if errors.Is(err, ErrConcurrentUpdate) { + h.writeError(w, http.StatusConflict, ErrCodeConflict, "table bucket changed during delete, retry the request") + } else { + h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table bucket") + } return err } + bucketClaim = "" h.writeJSON(w, http.StatusOK, nil) return nil diff --git a/weed/s3api/s3tables/handler_bucket_keeps_data_test.go b/weed/s3api/s3tables/handler_bucket_keeps_data_test.go new file mode 100644 index 0000000000..dfeef51285 --- /dev/null +++ b/weed/s3api/s3tables/handler_bucket_keeps_data_test.go @@ -0,0 +1,997 @@ +package s3tables + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "path" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// startEmptyBucket seeds a table bucket with nothing in it, not even a namespace. +func startEmptyBucket(t *testing.T) (*s3tablestest.MemFiler, *Manager) { + t.Helper() + fs := s3tablestest.Start(t) + bucketMeta, err := json.Marshal(tableBucketMetadata{Name: renameTestBucket, OwnerAccountID: DefaultAccountID}) + require.NoError(t, err) + fs.Put(TablesPath, renameTestBucket, map[string][]byte{ + ExtendedKeyTableBucket: []byte("{}"), + ExtendedKeyMetadata: bucketMeta, + }) + m := NewManager() + m.SetTrusted(true) + return fs, m +} + +func deleteTableBucketRequest(t *testing.T) *DeleteTableBucketRequest { + return &DeleteTableBucketRequest{TableBucketARN: mustBucketARN(t)} +} + +// Deleting a namespace that holds renamed-away data leaves its directory behind +// with no record. Once the renamed table is dropped the data is gone, and the +// directory is an empty leftover that must not keep the bucket from being deleted. +func TestDeleteTableBucketRemovesEmptyLeftovers(t *testing.T) { + fs, m := startDeletedNamespaceWithRenamedData(t) + bucketPath := GetTableBucketPath(renameTestBucket) + require.NotNil(t, fs.Get(bucketPath, "ns"), "the delete of ns keeps its directory, which is the leftover under test") + + requireOK(t, m, fs, "DeleteTable", &DeleteTableRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns2"}, + Name: "t", + }) + requireOK(t, m, fs, "DeleteNamespace", &DeleteNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns2"}, + }) + require.Nil(t, fs.Get(keptDataDir(), keptDataFile), "dropping the renamed table purges the data the leftover kept") + require.NotNil(t, fs.Get(bucketPath, "ns"), "the leftover directory is still in the bucket") + + requireOK(t, m, fs, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Nil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is deleted") + assert.Nil(t, fs.Get(bucketPath, "ns"), "the empty leftover goes with it") +} + +// No table refers to a file under a directory with no record, but deleting the +// bucket would destroy it without anyone deciding to. The bucket stays, and so +// does the file. +func TestDeleteTableBucketRefusesOrphanData(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + orphan := path.Join(bucketPath, "old", "t", "data") + fs.Put(bucketPath, "old", nil) + fs.Put(path.Join(bucketPath, "old"), "t", nil) + fs.Put(path.Join(bucketPath, "old", "t"), "data", nil) + fs.PutFile(orphan, keptDataFile, time.Now()) + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + assert.Contains(t, resp.Message, "directory old still holds files", "the answer names the directory that holds the file") + assert.NotNil(t, fs.Get(orphan, keptDataFile), "the file must survive") + assert.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") +} + +// A hidden directory can hold data the same way a visible leftover does. +// Files under it refuse the delete; only a tree holding none goes with the +// bucket, and so does a hidden file. +func TestDeleteTableBucketKeepsDataUnderAHiddenDirectory(t *testing.T) { + cases := []struct { + name string + seed func(t *testing.T, fs *s3tablestest.MemFiler, bucketPath, hiddenPath string) + kept bool + }{ + {"files under a hidden directory", func(t *testing.T, fs *s3tablestest.MemFiler, bucketPath, hiddenPath string) { + fs.Put(bucketPath, ".hidden", nil) + fs.Put(hiddenPath, "data", nil) + fs.PutFile(path.Join(hiddenPath, "data"), keptDataFile, time.Now()) + }, true}, + {"only empty hidden directories", func(t *testing.T, fs *s3tablestest.MemFiler, bucketPath, hiddenPath string) { + fs.Put(bucketPath, ".hidden", nil) + fs.Put(hiddenPath, "data", nil) + }, false}, + {"a hidden file", func(t *testing.T, fs *s3tablestest.MemFiler, bucketPath, hiddenPath string) { + fs.PutFile(bucketPath, ".DS_Store", time.Now()) + }, false}, + {"a hidden directory stored as an object", func(t *testing.T, fs *s3tablestest.MemFiler, bucketPath, hiddenPath string) { + putDirectoryObject(t, fs, bucketPath, ".hidden") + }, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + tc.seed(t, fs, bucketPath, path.Join(bucketPath, ".hidden")) + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + if tc.kept { + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + assert.NotNil(t, fs.Get(bucketPath, ".hidden"), "the hidden entry must survive") + assert.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") + } else { + assert.Equal(t, http.StatusOK, status, "%+v", resp) + assert.Nil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is deleted") + } + }) + } +} + +// pauseListClient runs pause just before the nth listing of dir, so a test can +// land a competing write at a chosen step of a delete's directory walk. +type pauseListClient struct { + filer_pb.SeaweedFilerClient + dir string + nth int + listings int + pause func() +} + +func (c *pauseListClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) { + if in.Directory == c.dir { + c.listings++ + if c.listings == c.nth { + c.pause() + } + } + return c.SeaweedFilerClient.ListEntries(ctx, in, opts...) +} + +func createNamespaceNsRequest(t *testing.T) *CreateNamespaceRequest { + return &CreateNamespaceRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ns"}} +} + +// A create could take a leftover directory back after the scan listed it, and a +// namespace with nothing in it holds no file for the walk to find, so the +// recursive delete would remove a namespace the create had already answered +// for. The delete's claim on the bucket turns the create away instead. +func TestDeleteTableBucketTurnsAwayATakeBackAfterTheScan(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + fs.Put(bucketPath, "ns", nil) + var createStatus int + var createResp S3TablesError + // The scan lists the bucket again for the page after the leftover, so the + // create lands once the scan has recorded it and before the delete walks it. + client := &pauseListClient{SeaweedFilerClient: fs.Client, dir: bucketPath, nth: 2} + client.pause = func() { + createStatus, createResp = runHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + } + + status, resp := runHandler(t, m, client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + require.GreaterOrEqual(t, client.listings, 2, "the create has to land after the scan listed the leftover") + assert.Equal(t, http.StatusConflict, createStatus, "%+v", createResp) + assert.Equal(t, ErrCodeConflict, createResp.Type, "%+v", createResp) + assert.Equal(t, http.StatusOK, status, "%+v", resp) + assert.Nil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is deleted") +} + +// A create that finds the bucket claimed by a delete is refused and writes +// nothing, as a create under a claimed namespace is. +func TestCreateNamespaceRefusedInClaimedBucket(t *testing.T) { + fs, m := startEmptyBucket(t) + plantDeleteClaim(fs, TablesPath, renameTestBucket) + + status, resp := runHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + assert.Nil(t, fs.Get(GetTableBucketPath(renameTestBucket), "ns"), "a refused create writes nothing") +} + +// A create is authorized against the bucket it read. If that bucket is deleted +// and another is created under the same name before the create writes, the +// namespace would land in a bucket nobody authorized it for, carrying the first +// bucket's owner. The create checks the bucket by its creation time, reads the +// new one as the old one gone, and writes nothing into it. +func TestCreateNamespaceTreatsARecreatedBucketAsGone(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + recreatedMeta, err := json.Marshal(tableBucketMetadata{ + Name: renameTestBucket, + OwnerAccountID: "111122223333", + CreatedAt: time.Now(), + }) + require.NoError(t, err) + recreated := map[string][]byte{ + ExtendedKeyTableBucket: []byte("{}"), + ExtendedKeyMetadata: recreatedMeta, + } + // The create looks its namespace up right before it writes; the bucket is + // replaced there. + fs.OnLookup = func(dir, name string) { + if dir != bucketPath || name != "ns" { + return + } + fs.OnLookup = nil + fs.Put(TablesPath, renameTestBucket, recreated) + } + + status, resp := runHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + + assert.Equal(t, http.StatusNotFound, status, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchBucket, resp.Type, "%+v", resp) + assert.Nil(t, fs.Get(bucketPath, "ns"), "nothing is written into the new bucket") + require.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the new bucket stays") + assert.Equal(t, recreated, fs.Get(TablesPath, renameTestBucket).Extended, "and is left as it was created") +} + +// A delete is authorized against the bucket it read. If that bucket is deleted +// and another is created under the same name before the delete claims it, the +// claim and the removal would act on a bucket nobody authorized the delete for. +// The claim takes only the bucket with the creation time the delete read, so +// the new one reads as the old one gone and is left as it was. +func TestDeleteTableBucketTreatsARecreatedBucketAsGone(t *testing.T) { + fs, m := startEmptyBucket(t) + recreatedMeta, err := json.Marshal(tableBucketMetadata{ + Name: renameTestBucket, + OwnerAccountID: "111122223333", + CreatedAt: time.Now(), + }) + require.NoError(t, err) + recreated := map[string][]byte{ + ExtendedKeyTableBucket: []byte("{}"), + ExtendedKeyMetadata: recreatedMeta, + } + // The claim is the delete's first write. The bucket is replaced as it lands, + // so the write loses and its retry reads the new bucket. + fs.BeforeUpdate = func() { fs.Put(TablesPath, renameTestBucket, recreated) } + + code, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusNotFound, code, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchBucket, resp.Type, "%+v", resp) + require.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the new bucket stays") + assert.Equal(t, recreated, fs.Get(TablesPath, renameTestBucket).Extended, "and is left as it was created") +} + +// failNthLookupClient answers the nth lookup of target with Unavailable, the +// way a filer that is briefly unreachable would, and counts the lookups of the +// entries under target that follow. +type failNthLookupClient struct { + filer_pb.SeaweedFilerClient + target string + nth int + lookups int + childLookups int +} + +func (c *failNthLookupClient) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) { + switch path.Join(req.Directory, req.Name) { + case c.target: + c.lookups++ + if c.lookups == c.nth { + return nil, status.Error(codes.Unavailable, "filer unavailable") + } + default: + if req.Directory == c.target { + c.childLookups++ + } + } + return c.SeaweedFilerClient.LookupDirectoryEntry(ctx, req, opts...) +} + +// The check of the bucket before the namespace is written either passes or ends +// the create. A bucket it could not read, or one it finds still being created, +// is not one to write into: nothing is written, where a create that went on +// would write and leave its confirmation to back out. +func TestCreateNamespaceStopsAtAFailedBucketCheck(t *testing.T) { + bucketPath := GetTableBucketPath(renameTestBucket) + + t.Run("the bucket could not be read", func(t *testing.T) { + fs, m := startEmptyBucket(t) + // The create reads the bucket three times for its authorization; the + // fourth read is the check before the write. + client := &failNthLookupClient{SeaweedFilerClient: fs.Client, target: bucketPath, nth: 4} + + code, resp := runHandler(t, m, client, "CreateNamespace", createNamespaceNsRequest(t)) + + assert.Equal(t, http.StatusInternalServerError, code, "%+v", resp) + assert.Equal(t, ErrCodeInternalError, resp.Type, "%+v", resp) + assert.Zero(t, client.childLookups, "the create stops before it looks at the namespace") + assert.Nil(t, fs.Get(bucketPath, "ns"), "nothing is written") + }) + t.Run("the bucket is still being created", func(t *testing.T) { + fs, m := startEmptyBucket(t) + plantCreateMark(fs, TablesPath, renameTestBucket) + + code, resp := runHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + assert.Nil(t, fs.Get(bucketPath, "ns"), "nothing is written") + }) +} + +// A delete that has scanned an empty bucket must not lose to a create that +// lands before the bucket is removed: the claim the delete holds since before +// the scan turns the create away, so no namespace is answered for and deleted. +func TestDeleteTableBucketTurnsAwayACreateAfterTheScan(t *testing.T) { + fs, m := startEmptyBucket(t) + var createStatus int + var createResp S3TablesError + fs.BeforeDelete = func() { + createStatus, createResp = runHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + } + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + require.NotZero(t, createStatus, "the create has to land between the scan and the delete") + assert.Equal(t, http.StatusConflict, createStatus, "%+v", createResp) + assert.Equal(t, ErrCodeConflict, createResp.Type, "%+v", createResp) + assert.Equal(t, http.StatusOK, status, "%+v", resp) + assert.Nil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is deleted") +} + +// A create can pass the claim check before the delete claims the bucket, and +// then write a namespace that was not there when the delete scanned. The scan +// never saw it, and the recursive delete would remove it after the create had +// answered. The create's confirmation after its write finds the claim, so it +// backs out and reports the conflict instead of success. +func TestCreateNamespaceWrittenAfterTheScanBacksOut(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + reached := make(chan struct{}) + release := make(chan struct{}) + // The create looks its namespace up right after the claim check; hold it there. + fs.OnLookup = func(dir, name string) { + if dir != bucketPath || name != "ns" { + return + } + fs.OnLookup = nil + close(reached) + <-release + } + created := startHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + waitForPause(t, reached, created) + // The delete claims the bucket and scans it empty. The create writes only + // now, before the delete removes the bucket. + var createResult handlerResult + fs.BeforeDelete = func() { + close(release) + createResult = <-created + } + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + require.NotZero(t, createResult.status, "the create has to write between the scan and the delete") + assert.Equal(t, http.StatusConflict, createResult.status, "%+v", createResult.resp) + assert.Equal(t, ErrCodeConflict, createResult.resp.Type, "%+v", createResult.resp) + assert.Equal(t, http.StatusOK, status, "%+v", resp) + assert.Nil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is deleted") + assert.Nil(t, fs.Get(bucketPath, "ns"), "no namespace the create answered for is left behind") +} + +// The claim a delete takes on the bucket can be replaced by a second delete, +// which lifts it when it refuses. A create then finds the bucket open, and the +// first delete, which scanned before any of that, must not remove the bucket +// on a claim it no longer holds: it would take the namespace the create +// answered for, and the file under it, with it. It answers 409 and leaves both. +func TestDeleteTableBucketRefusesAfterItsClaimIsLifted(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + nsPath := path.Join(bucketPath, "ns") + var refusedStatus, createStatus int + var refusedResp, createResp S3TablesError + // The first delete has scanned an empty bucket when it reaches its removal. + fs.BeforeDelete = func() { + // A directory with a file, kept by a namespace that was deleted, turns up + // and the second delete refuses on it, lifting its own claim. + fs.Put(bucketPath, "ns", nil) + fs.PutFile(nsPath, keptDataFile, time.Now()) + refusedStatus, refusedResp = runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + // The bucket is open again, so the create takes the directory back. + createStatus, createResp = runHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + } + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + require.NotZero(t, createStatus, "the create has to land between the first delete's scan and its removal") + assert.Equal(t, http.StatusConflict, refusedStatus, "%+v", refusedResp) + assert.Equal(t, ErrCodeBucketNotEmpty, refusedResp.Type, "%+v", refusedResp) + assert.Equal(t, http.StatusOK, createStatus, "%+v", createResp) + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + require.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") + assert.NotContains(t, fs.Get(TablesPath, renameTestBucket).Extended, ExtendedKeyDeleting) + assert.NotNil(t, fs.Get(bucketPath, "ns"), "the namespace the create answered for must survive") + assert.NotNil(t, fs.Get(nsPath, keptDataFile), "the file under it must survive") +} + +// The walk that finds a leftover directory empty is over before the bucket is +// removed, and a file can be written in between: an upload through the S3 API +// takes no claim. Nothing the removal does may take a file it was not shown, so +// it is refused at the directory that holds one, and a file written into the +// bucket itself makes the removal of the bucket refuse the same way. +func TestDeleteTableBucketKeepsAFileUploadedAfterTheWalk(t *testing.T) { + for _, tc := range []struct { + name string + dir func(bucketPath string) string + }{ + {"into a leftover directory", func(bucketPath string) string { return path.Join(bucketPath, "old") }}, + {"into the bucket", func(bucketPath string) string { return bucketPath }}, + } { + t.Run(tc.name, func(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + fs.Put(bucketPath, "old", nil) + uploadDir := tc.dir(bucketPath) + fs.BeforeDelete = func() { + fs.PutFile(uploadDir, keptDataFile, time.Now()) + } + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + assert.NotNil(t, fs.Get(uploadDir, keptDataFile), "the uploaded file must survive") + require.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") + assert.NotContains(t, fs.Get(TablesPath, renameTestBucket).Extended, ExtendedKeyDeleting, "a refused delete lifts its claim") + if uploadDir != bucketPath { + require.NotNil(t, fs.Get(bucketPath, "old")) + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "and its claim on the leftover") + } + }) + } +} + +// A second delete can replace the first one's claim and lift it, and a create +// then takes a leftover directory back as a namespace after the first delete +// scanned it bare, and an empty table is created in it. Neither holds a file, +// so only the attributes the scan saw on the directory tell the removal it is +// no longer the same directory, and that has to be settled before anything +// under it goes: the delete refuses with a conflict and leaves both. +func TestDeleteTableBucketKeepsALeftoverTakenBackAfterItsClaimIsLifted(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + nsPath := path.Join(bucketPath, "ns") + fs.Put(bucketPath, "ns", nil) + var refusedStatus, createStatus, tableStatus int + var refusedResp, createResp, tableResp S3TablesError + fs.BeforeDelete = func() { + // A file under another leftover makes the second delete refuse, and + // lift its claim; the bucket is open again for the create. + fs.Put(bucketPath, "old", nil) + fs.PutFile(path.Join(bucketPath, "old"), keptDataFile, time.Now()) + refusedStatus, refusedResp = runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + createStatus, createResp = runHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + tableStatus, tableResp = runHandler(t, m, fs.Client, "CreateTable", createTableRequest(t, "ns", "t")) + } + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + require.NotZero(t, tableStatus, "the creates have to land between the first delete's scan and its removal") + assert.Equal(t, http.StatusConflict, refusedStatus, "%+v", refusedResp) + assert.Equal(t, http.StatusOK, createStatus, "%+v", createResp) + assert.Equal(t, http.StatusOK, tableStatus, "%+v", tableResp) + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + require.NotNil(t, fs.Get(bucketPath, "ns"), "the namespace the create answered for must survive") + assert.Contains(t, fs.Get(bucketPath, "ns").Extended, ExtendedKeyMetadata) + require.NotNil(t, fs.Get(nsPath, "t"), "the table the create answered for must survive") + assert.Contains(t, fs.Get(nsPath, "t").Extended, ExtendedKeyMetadata) + assert.NotNil(t, fs.Get(path.Join(nsPath, "t"), "data"), "and so must its data directory") + assert.NotNil(t, fs.Get(path.Join(bucketPath, "old"), keptDataFile), "the file under the other leftover must survive") + assert.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") +} + +// A catalog record under a leftover directory is not the leftover's to remove, +// whatever it holds: an empty table is still a table. The delete refuses at +// it, and lifts the claim it took on the leftover. +func TestDeleteTableBucketRefusesACatalogRecordInALeftover(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + leftover := path.Join(bucketPath, "old") + fs.Put(bucketPath, "old", nil) + fs.Put(leftover, "t", map[string][]byte{ + ExtendedKeyMetadata: []byte("{}"), + ExtendedKeyEntryType: []byte(EntryTypeTable), + }) + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + require.NotNil(t, fs.Get(leftover, "t"), "the record must survive") + assert.Contains(t, fs.Get(leftover, "t").Extended, ExtendedKeyMetadata) + require.NotNil(t, fs.Get(bucketPath, "old")) + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "a refused delete lifts its claim on the leftover") + assert.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") +} + +// A namespace a create left marked unfinished must not wedge the name: the mark +// says its writer never confirmed it, so the next create takes it over, and +// tables can be created in it once it is confirmed. +func TestCreateNamespaceAdoptsAnUnfinishedNamespace(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + plantCreateMark(fs, bucketPath, "ns") + + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + + assert.NotContains(t, fs.Get(bucketPath, "ns").Extended, ExtendedKeyDeleting, "the adopted namespace ends confirmed") + requireOK(t, m, fs, "CreateTable", createTableRequest(t, "ns", "t")) +} + +// A table answers success once its namespace is open, and says nothing of the +// bucket, so a table created while the namespace's own create still has to +// confirm the bucket would survive as a table under a namespace that +// confirmation backs out. The namespace carries the unfinished mark until the +// confirmation has passed, and is absent to a create under it meanwhile. +func TestCreateTableRefusedInNamespaceStillBeingCreated(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + reached := make(chan struct{}) + release := make(chan struct{}) + // The create confirms the bucket right after writing the namespace; hold it there. + fs.OnLookup = func(dir, name string) { + if dir != TablesPath || name != renameTestBucket || fs.Get(bucketPath, "ns") == nil { + return + } + fs.OnLookup = nil + close(reached) + <-release + } + created := startHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + waitForPause(t, reached, created) + + status, resp := runHandler(t, m, fs.Client, "CreateTable", createTableRequest(t, "ns", "t")) + close(release) + createResult := <-created + + assert.Equal(t, http.StatusNotFound, status, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchNamespace, resp.Type, "%+v", resp) + assert.Nil(t, fs.Get(path.Join(bucketPath, "ns"), "t"), "a refused create writes nothing") + assert.Equal(t, http.StatusOK, createResult.status, "%+v", createResult.resp) + assert.NotContains(t, fs.Get(bucketPath, "ns").Extended, ExtendedKeyDeleting, "a confirmed namespace carries no mark") + requireOK(t, m, fs, "CreateTable", createTableRequest(t, "ns", "t")) +} + +// A namespace whose create has not confirmed it is absent to every reader, as +// a table's is. A delete that claimed it would replace the create's mark, and +// lifting its own claim on a refusal would leave the namespace looking +// confirmed: open to tables before its bucket check passed, and out of reach of +// the create's back-out. Here the create takes back a leftover holding a file, +// which a delete would refuse on. +func TestDeleteNamespaceLeavesAnUnfinishedNamespaceAlone(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + nsPath := path.Join(bucketPath, "ns") + fs.Put(bucketPath, "ns", nil) + fs.PutFile(nsPath, keptDataFile, time.Now()) + reached := make(chan struct{}) + release := make(chan struct{}) + // The create confirms the bucket right after writing the namespace; hold it there. + fs.OnLookup = func(dir, name string) { + if dir != TablesPath || name != renameTestBucket { + return + } + if entry := fs.Get(bucketPath, "ns"); entry == nil || entry.Extended[ExtendedKeyMetadata] == nil { + return + } + fs.OnLookup = nil + close(reached) + <-release + } + created := startHandler(t, m, fs.Client, "CreateNamespace", createNamespaceNsRequest(t)) + waitForPause(t, reached, created) + + deleteStatus, deleteResp := runHandler(t, m, fs.Client, "DeleteNamespace", + &DeleteNamespaceRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ns"}}) + listed := listedNamespaces(t, m, fs) + tableStatus, tableResp := runHandler(t, m, fs.Client, "CreateTable", createTableRequest(t, "ns", "t")) + close(release) + createResult := <-created + + assert.Equal(t, http.StatusNotFound, deleteStatus, "%+v", deleteResp) + assert.Equal(t, ErrCodeNoSuchNamespace, deleteResp.Type, "%+v", deleteResp) + assert.NotContains(t, listed, "ns", "an unfinished namespace is not listed") + assert.Equal(t, http.StatusNotFound, tableStatus, "%+v", tableResp) + assert.Equal(t, ErrCodeNoSuchNamespace, tableResp.Type, "%+v", tableResp) + assert.Nil(t, fs.Get(nsPath, "t"), "no table is created in it") + assert.Equal(t, http.StatusOK, createResult.status, "%+v", createResult.resp) + assert.NotContains(t, fs.Get(bucketPath, "ns").Extended, ExtendedKeyDeleting, "the create confirms it") + assert.NotNil(t, fs.Get(nsPath, keptDataFile), "the file under it stays") +} + +// A namespace can become unfinished between a delete's read and its claim: a +// create took its name back meanwhile. The claim refuses it as absent, rather +// than replacing the create's mark. +func TestDeleteNamespaceDoesNotClaimAnUnfinishedNamespace(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + // The claim is the delete's first write. + fs.BeforeUpdate = func() { plantCreateMark(fs, bucketPath, "ns") } + + status, resp := runHandler(t, m, fs.Client, "DeleteNamespace", + &DeleteNamespaceRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"ns"}}) + + assert.Equal(t, http.StatusNotFound, status, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchNamespace, resp.Type, "%+v", resp) + require.NotNil(t, fs.Get(bucketPath, "ns"), "the namespace stays") + assert.Equal(t, []byte(createClaim), fs.Get(bucketPath, "ns").Extended[ExtendedKeyDeleting], "with its create's mark") +} + +// A delete that claims the bucket between the namespace's confirmation and the +// end of its create must still see the namespace, or it would remove a +// namespace the create answered for. The mark does not hide it from the scan. +func TestDeleteTableBucketCountsANamespaceStillBeingCreated(t *testing.T) { + fs, m := startEmptyBucket(t) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + plantCreateMark(fs, GetTableBucketPath(renameTestBucket), "ns") + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + assert.Contains(t, resp.Message, "namespace ns", "the hidden namespace is named") + assert.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is kept") +} + +// The claim on the bucket is lifted whenever the delete refuses, or the bucket +// would refuse every create; a claim left by a delete that died does not keep +// the next delete from running. +func TestDeleteTableBucketClaimOnTheBucket(t *testing.T) { + lifted := func(t *testing.T, fs *s3tablestest.MemFiler, m *Manager) { + t.Helper() + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.NotContains(t, fs.Get(TablesPath, renameTestBucket).Extended, ExtendedKeyDeleting) + requireOK(t, m, fs, "CreateNamespace", &CreateNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"other"}, + }) + } + t.Run("lifted when a namespace is found", func(t *testing.T) { + fs, m := startEmptyBucket(t) + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) + lifted(t, fs, m) + }) + t.Run("lifted when a directory holds files", func(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + fs.Put(bucketPath, "old", nil) + fs.PutFile(path.Join(bucketPath, "old"), keptDataFile, time.Now()) + lifted(t, fs, m) + }) + t.Run("replaces the claim of a delete that died", func(t *testing.T) { + fs, m := startEmptyBucket(t) + plantDeleteClaim(fs, TablesPath, renameTestBucket) + + requireOK(t, m, fs, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Nil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is deleted") + }) +} + +// putDirectoryObject stores an S3 object with content on the directory entry +// dir/name, the way the S3 API stores a key that other keys sit under: with the +// prefix-object mark the filer expects on an object landing on a directory. +func putDirectoryObject(t *testing.T, fs *s3tablestest.MemFiler, dir, name string) { + t.Helper() + _, err := fs.Client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{ + Directory: dir, + Entry: &filer_pb.Entry{ + Name: name, + IsDirectory: true, + Attributes: &filer_pb.FuseAttributes{Mime: "application/octet-stream"}, + Content: []byte("object"), + Extended: map[string][]byte{s3_constants.SeaweedFSPrefixObject: []byte("true")}, + }, + }) + require.NoError(t, err) +} + +// markPrefixObject marks the directory dir/name as also holding an empty S3 +// object, which leaves no trace but the mark, keeping its other attributes. +func markPrefixObject(fs *s3tablestest.MemFiler, dir, name string) { + extended := map[string][]byte{} + for key, value := range fs.Get(dir, name).Extended { + extended[key] = value + } + extended[s3_constants.SeaweedFSPrefixObject] = []byte("true") + fs.Put(dir, name, extended) +} + +// beforeDeleteClient runs before just ahead of the first delete of target. +type beforeDeleteClient struct { + filer_pb.SeaweedFilerClient + target string + before func() +} + +func (c *beforeDeleteClient) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest, opts ...grpc.CallOption) (*filer_pb.DeleteEntryResponse, error) { + if before := c.before; before != nil && path.Join(req.Directory, req.Name) == c.target { + c.before = nil + before() + } + return c.SeaweedFilerClient.DeleteEntry(ctx, req, opts...) +} + +// An S3 object can be stored on a directory entry: a key other keys sit under, +// or an empty object that leaves nothing but a mark on the directory. Such a +// directory holds no file, yet it is user data, not a leftover: the delete +// refuses wherever it finds one and the object stays. +func TestDeleteTableBucketKeepsADirectoryBackedObject(t *testing.T) { + bucketPath := GetTableBucketPath(renameTestBucket) + leftover := path.Join(bucketPath, "old") + refused := func(t *testing.T, code int, resp S3TablesError, wantType string) { + t.Helper() + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, wantType, resp.Type, "%+v", resp) + } + + t.Run("an empty object in the bucket", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", map[string][]byte{s3_constants.SeaweedFSPrefixObject: []byte("true")}) + + code, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + refused(t, code, resp, ErrCodeBucketNotEmpty) + require.NotNil(t, fs.Get(bucketPath, "old"), "the object must survive") + assert.True(t, fs.Get(bucketPath, "old").IsPrefixObject()) + }) + t.Run("an object under a leftover directory", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + putDirectoryObject(t, fs, leftover, "a") + + code, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + refused(t, code, resp, ErrCodeBucketNotEmpty) + require.NotNil(t, fs.Get(leftover, "a"), "the object must survive") + assert.Equal(t, []byte("object"), fs.Get(leftover, "a").Content) + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "a refused delete lifts its claim on the leftover") + }) + t.Run("a leftover stored as an object after the scan", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + fs.BeforeDelete = func() { putDirectoryObject(t, fs, bucketPath, "old") } + + code, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + refused(t, code, resp, ErrCodeBucketNotEmpty) + require.NotNil(t, fs.Get(bucketPath, "old"), "the object must survive") + assert.Equal(t, []byte("object"), fs.Get(bucketPath, "old").Content) + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "no claim is taken on an object") + }) + t.Run("a leftover stored as an object between the read and the write of its claim", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + // The claim read the bare directory; the object lands before its write, + // which would otherwise put the bare directory back over it. + fs.OnUpdate = func(dir, name string) { + if dir != bucketPath || name != "old" { + return + } + fs.OnUpdate = nil + putDirectoryObject(t, fs, bucketPath, "old") + } + + code, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + refused(t, code, resp, ErrCodeBucketNotEmpty) + require.NotNil(t, fs.Get(bucketPath, "old"), "the object must survive") + assert.Equal(t, []byte("object"), fs.Get(bucketPath, "old").Content) + assert.True(t, fs.Get(bucketPath, "old").IsPrefixObject()) + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "no claim is taken on an object") + }) + t.Run("a leftover marked as an empty object after the walk", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + client := &beforeDeleteClient{SeaweedFilerClient: fs.Client, target: leftover} + client.before = func() { markPrefixObject(fs, bucketPath, "old") } + + code, resp := runHandler(t, m, client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + refused(t, code, resp, ErrCodeConflict) + require.NotNil(t, fs.Get(bucketPath, "old"), "the object must survive") + assert.True(t, fs.Get(bucketPath, "old").IsPrefixObject()) + }) +} + +// A directory the delete listed can be removed by another client and a file +// uploaded at its path before the delete reaches it. The file carries no +// attribute a directory would not, so only the directory's type and the claim +// written onto it tell the two apart: the delete refuses and the file stays, +// whether it replaced the leftover itself or a directory under it, and whether +// it landed before that directory was claimed or after. +func TestDeleteTableBucketKeepsAFileThatReplacedADirectory(t *testing.T) { + bucketPath := GetTableBucketPath(renameTestBucket) + leftover := path.Join(bucketPath, "old") + kept := func(t *testing.T, fs *s3tablestest.MemFiler, dir, name string) { + t.Helper() + entry := fs.Get(dir, name) + require.NotNil(t, entry, "the uploaded file must survive") + assert.False(t, entry.IsDirectory) + assert.NotContains(t, entry.Extended, ExtendedKeyDeleting, "no claim is left on the file") + require.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") + } + + t.Run("the leftover, after the scan", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + fs.BeforeDelete = func() { fs.PutFile(bucketPath, "old", time.Now()) } + + code, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + kept(t, fs, bucketPath, "old") + }) + t.Run("a directory under it, after the walk listed it", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + fs.Put(leftover, "child", nil) + // The scan's look for files and the removal's walk each list old, then + // old/child. The file takes child's place as the walk reaches it, after + // the walk listed child as a directory and before child is claimed. + client := &pauseListClient{SeaweedFilerClient: fs.Client, dir: path.Join(leftover, "child"), nth: 2} + client.pause = func() { fs.PutFile(leftover, "child", time.Now()) } + + code, resp := runHandler(t, m, client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + require.Equal(t, 2, client.listings, "the file has to land during the removal's walk") + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + kept(t, fs, leftover, "child") + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "a refused delete lifts its claim on the leftover") + }) + t.Run("a directory under it, between the read and the write of its claim", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + fs.Put(leftover, "child", nil) + // The filer refuses to write the claimed directory over the file and + // says so only as an opaque failure. + fs.OnUpdate = func(dir, name string) { + if dir != leftover || name != "child" { + return + } + fs.OnUpdate = nil + fs.PutFile(leftover, "child", time.Now()) + } + + code, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + kept(t, fs, leftover, "child") + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "a refused delete lifts its claim on the leftover") + }) + t.Run("a directory under it, after its claim", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + fs.Put(leftover, "child", nil) + client := &beforeDeleteClient{SeaweedFilerClient: fs.Client, target: path.Join(leftover, "child")} + client.before = func() { fs.PutFile(leftover, "child", time.Now()) } + + code, resp := runHandler(t, m, client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + require.Nil(t, client.before, "the file has to land right before the delete") + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + kept(t, fs, leftover, "child") + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "a refused delete lifts its claim on the leftover") + }) +} + +// failClaimReleaseClient answers the first write that lifts a delete claim from +// target with Unavailable, the way a filer that is briefly unreachable would. +type failClaimReleaseClient struct { + filer_pb.SeaweedFilerClient + target string + failed int +} + +func (c *failClaimReleaseClient) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest, opts ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) { + if c.failed == 0 && req.Entry != nil && path.Join(req.Directory, req.Entry.Name) == c.target && + req.Entry.Extended[ExtendedKeyDeleting] == nil && len(req.ExpectedExtended[ExtendedKeyDeleting]) > 0 { + c.failed++ + return nil, status.Error(codes.Unavailable, "filer unavailable") + } + return c.SeaweedFilerClient.UpdateEntry(ctx, req, opts...) +} + +// A claim the delete fails to lift refuses every create in the bucket until the +// next delete replaces it, so a release the filer turns away once is retried. +func TestDeleteTableBucketRetriesTheClaimRelease(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + fs.Put(bucketPath, "old", nil) + fs.PutFile(path.Join(bucketPath, "old"), keptDataFile, time.Now()) + client := &failClaimReleaseClient{SeaweedFilerClient: fs.Client, target: bucketPath} + + code, resp := runHandler(t, m, client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + require.Equal(t, 1, client.failed, "the first release has to fail") + assert.NotContains(t, fs.Get(TablesPath, renameTestBucket).Extended, ExtendedKeyDeleting, "a later attempt lifts the claim") + requireOK(t, m, fs, "CreateNamespace", createNamespaceNsRequest(t)) +} + +// A claim the removal fails to lift from a leftover directory makes a create +// read the leftover's name as taken, and one left on a directory under it counts +// as a child of the namespace the leftover may become. A release the filer turns +// away once is retried for both. +func TestDeleteTableBucketRetriesLeftoverClaimReleases(t *testing.T) { + bucketPath := GetTableBucketPath(renameTestBucket) + leftover := path.Join(bucketPath, "old") + + t.Run("on the leftover", func(t *testing.T) { + fs, m := startEmptyBucket(t) + fs.Put(bucketPath, "old", nil) + failing := &failClaimReleaseClient{SeaweedFilerClient: fs.Client, target: leftover} + // A file lands in the leftover after the scan, so the walk refuses it and + // the leftover's claim has to come off. + client := &beforeDeleteClient{SeaweedFilerClient: failing, target: GetTableObjectBucketPath(renameTestBucket)} + client.before = func() { fs.PutFile(leftover, keptDataFile, time.Now()) } + + code, resp := runHandler(t, m, client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + require.Equal(t, 1, failing.failed, "the first release has to fail") + assert.NotContains(t, fs.Get(bucketPath, "old").Extended, ExtendedKeyDeleting, "a later attempt lifts the claim") + requireOK(t, m, fs, "CreateNamespace", &CreateNamespaceRequest{TableBucketARN: mustBucketARN(t), Namespace: []string{"old"}}) + }) + t.Run("on a directory under it", func(t *testing.T) { + fs, m := startEmptyBucket(t) + child := path.Join(leftover, "child") + fs.Put(bucketPath, "old", nil) + fs.Put(leftover, "child", nil) + failing := &failClaimReleaseClient{SeaweedFilerClient: fs.Client, target: child} + // A file lands in child after its claim, so its delete is refused and + // the claim has to come off. + client := &beforeDeleteClient{SeaweedFilerClient: failing, target: child} + client.before = func() { fs.PutFile(child, keptDataFile, time.Now()) } + + code, resp := runHandler(t, m, client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + require.Equal(t, 1, failing.failed, "the first release has to fail") + assert.NotContains(t, fs.Get(leftover, "child").Extended, ExtendedKeyDeleting, "a later attempt lifts the claim") + assert.NotNil(t, fs.Get(child, keptDataFile), "the file stays") + }) +} + +// The scan reads the bucket a page at a time, and the leftovers it collects +// along the way do not decide anything. With more bare directories than one page +// holds, the namespace sorted after them is on a later page: stopping at the +// first page would find nothing but leftovers and delete the bucket with it. +func TestDeleteTableBucketFindsANamespacePastLeftovers(t *testing.T) { + fs, m := startEmptyBucket(t) + bucketPath := GetTableBucketPath(renameTestBucket) + for i := 0; i <= 10; i++ { + fs.Put(bucketPath, fmt.Sprintf("a%02d", i), nil) + } + requireOK(t, m, fs, "CreateNamespace", &CreateNamespaceRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"zz"}, + }) + + status, resp := runHandler(t, m, fs.Client, "DeleteTableBucket", deleteTableBucketRequest(t)) + + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeBucketNotEmpty, resp.Type, "%+v", resp) + assert.NotNil(t, fs.Get(bucketPath, "zz"), "the namespace must survive") + assert.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket must survive") +} diff --git a/weed/s3api/s3tables/handler_delete_create_race_test.go b/weed/s3api/s3tables/handler_delete_create_race_test.go index 38f075d3a9..7f51145d10 100644 --- a/weed/s3api/s3tables/handler_delete_create_race_test.go +++ b/weed/s3api/s3tables/handler_delete_create_race_test.go @@ -512,6 +512,25 @@ func TestUpdateRefusedOnAMarkedRecord(t *testing.T) { assert.Equal(t, ErrCodeNoSuchTable, resp.Type) } +// DeleteTablePolicy refuses a marked record the same way: the policy it would +// remove sits on an entry every reader reports absent. +func TestDeleteTablePolicyRefusedOnAMarkedRecord(t *testing.T) { + fs, m := startTableT(t, map[string][]byte{ExtendedKeyPolicy: []byte(bindTestPolicy)}) + nsPath := GetNamespacePath(renameTestBucket, "ns") + fs.BeforeUpdate = func() { plantCreateMark(fs, nsPath, "t") } + + status, resp := runHandler(t, m, fs.Client, "DeleteTablePolicy", &DeleteTablePolicyRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns"}, + Name: "t", + }) + assert.Equal(t, http.StatusNotFound, status, "%+v", resp) + assert.Equal(t, ErrCodeNoSuchTable, resp.Type) + extended := fs.Get(nsPath, "t").Extended + assert.Equal(t, []byte(bindTestPolicy), extended[ExtendedKeyPolicy], "the refused delete keeps the policy") + assert.Equal(t, []byte(createClaim), extended[ExtendedKeyDeleting], "and leaves the create's mark for its back-out") +} + // A rename into a namespace a delete claimed is refused before it touches the // source: the destination cannot be confirmed, so the name must stay where it // was. @@ -647,6 +666,22 @@ func TestDeleteNamespaceWithTableLiftsItsClaim(t *testing.T) { } } +// A claim the delete fails to lift refuses every create in the namespace, so a +// release the filer turns away once is retried. +func TestDeleteNamespaceRetriesTheClaimRelease(t *testing.T) { + fs, m := startRenameManager(t) + bucketPath := GetTableBucketPath(renameTestBucket) + before := fs.Get(bucketPath, "ns").Extended + client := &failClaimReleaseClient{SeaweedFilerClient: fs.Client, target: GetNamespacePath(renameTestBucket, "ns")} + + code, resp := runHandler(t, m, client, "DeleteNamespace", deleteNamespaceRequest(t)) + + assert.Equal(t, http.StatusConflict, code, "%+v", resp) + assert.Equal(t, ErrCodeNamespaceNotEmpty, resp.Type, "%+v", resp) + require.Equal(t, 1, client.failed, "the first release has to fail") + assert.Equal(t, before, fs.Get(bucketPath, "ns").Extended, "a later attempt lifts the claim") +} + // orphanMarkedRecord plants what a create that crashed after writing its // record leaves behind: catalog attributes under the unfinished mark. func orphanMarkedRecord(fs *s3tablestest.MemFiler, dir, name string) { diff --git a/weed/s3api/s3tables/handler_maintenance.go b/weed/s3api/s3tables/handler_maintenance.go index 02a2764a25..bd971e64cf 100644 --- a/weed/s3api/s3tables/handler_maintenance.go +++ b/weed/s3api/s3tables/handler_maintenance.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "time" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" ) @@ -45,12 +46,13 @@ func (h *S3TablesHandler) handlePutTableBucketMaintenanceConfiguration(w http.Re return err } - if _, err := h.authorizeMaintenanceBucket(r, filerClient, "PutTableBucketMaintenanceConfiguration", bucketName); err != nil { + _, createdAt, err := h.authorizeMaintenanceBucket(r, filerClient, "PutTableBucketMaintenanceConfiguration", bucketName) + if err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) return err } - if err := h.putMaintenanceConfiguration(r, filerClient, GetTableBucketPath(bucketName), req.Type, req.Value); err != nil { + if err := h.putMaintenanceConfiguration(r, filerClient, GetTableBucketPath(bucketName), req.Type, req.Value, requireTableBucketCreatedAt, createdAt); err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) return err } @@ -78,7 +80,7 @@ func (h *S3TablesHandler) handleGetTableBucketMaintenanceConfiguration(w http.Re return err } - bucketARN, err := h.authorizeMaintenanceBucket(r, filerClient, "GetTableBucketMaintenanceConfiguration", bucketName) + bucketARN, _, err := h.authorizeMaintenanceBucket(r, filerClient, "GetTableBucketMaintenanceConfiguration", bucketName) if err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) return err @@ -116,13 +118,14 @@ func (h *S3TablesHandler) handlePutTableMaintenanceConfiguration(w http.Response return err } - if _, err := h.authorizeMaintenanceTable(r, filerClient, "PutTableMaintenanceConfiguration", bucketName, namespaceName, tableName); err != nil { + _, createdAt, err := h.authorizeMaintenanceTable(r, filerClient, "PutTableMaintenanceConfiguration", bucketName, namespaceName, tableName) + if err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName)) return err } tablePath := GetTablePath(bucketName, namespaceName, tableName) - if err := h.putMaintenanceConfiguration(r, filerClient, tablePath, req.Type, req.Value); err != nil { + if err := h.putMaintenanceConfiguration(r, filerClient, tablePath, req.Type, req.Value, requireTableCreatedAt, createdAt); err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName)) return err } @@ -145,7 +148,7 @@ func (h *S3TablesHandler) handleGetTableMaintenanceConfiguration(w http.Response return err } - tableARN, err := h.authorizeMaintenanceTable(r, filerClient, "GetTableMaintenanceConfiguration", bucketName, namespaceName, tableName) + tableARN, _, err := h.authorizeMaintenanceTable(r, filerClient, "GetTableMaintenanceConfiguration", bucketName, namespaceName, tableName) if err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName)) return err @@ -181,7 +184,7 @@ func (h *S3TablesHandler) handleGetTableMaintenanceJobStatus(w http.ResponseWrit return err } - tableARN, err := h.authorizeMaintenanceTable(r, filerClient, "GetTableMaintenanceJobStatus", bucketName, namespaceName, tableName) + tableARN, _, err := h.authorizeMaintenanceTable(r, filerClient, "GetTableMaintenanceJobStatus", bucketName, namespaceName, tableName) if err != nil { h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName)) return err @@ -253,18 +256,25 @@ func buildJobStatusResponse(recorded MaintenanceJobStatus, config MaintenanceCon // putMaintenanceConfiguration merges one maintenance type into the stored // configuration, leaving the other types alone. The write checks the catalog // identity in the same conditional mutation, so it cannot land on a name a -// concurrent rename or delete has already soft-deleted. +// concurrent rename or delete has already soft-deleted. sameRecord refuses a +// record other than the one created at createdAt, which the caller was +// authorized against. func (h *S3TablesHandler) putMaintenanceConfiguration( r *http.Request, filerClient FilerClient, resourcePath, maintenanceType string, value *MaintenanceConfigurationValue, + sameRecord func(extended map[string][]byte, path string, createdAt time.Time) error, + createdAt time.Time, ) error { return filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, resourcePath, func(extended map[string][]byte) error { if len(extended[ExtendedKeyMetadata]) == 0 { return fmt.Errorf("%w: %s", filer_pb.ErrNotFound, resourcePath) } + if err := sameRecord(extended, resourcePath, createdAt); err != nil { + return err + } config := MaintenanceConfiguration{} if current := extended[ExtendedKeyMaintenance]; len(current) > 0 { @@ -325,8 +335,9 @@ func (h *S3TablesHandler) parseTableTarget(tableBucketARN string, namespace []st } // authorizeMaintenanceBucket checks the caller may perform operation on the -// table bucket, returning the bucket ARN. -func (h *S3TablesHandler) authorizeMaintenanceBucket(r *http.Request, filerClient FilerClient, operation, bucketName string) (string, error) { +// table bucket, returning the bucket ARN and the creation time of the bucket +// record the check read. +func (h *S3TablesHandler) authorizeMaintenanceBucket(r *http.Request, filerClient FilerClient, operation, bucketName string) (string, time.Time, error) { bucketPath := GetTableBucketPath(bucketName) var bucketMetadata tableBucketMetadata @@ -343,7 +354,7 @@ func (h *S3TablesHandler) authorizeMaintenanceBucket(r *http.Request, filerClien return err }) if err != nil { - return "", err + return "", time.Time{}, err } bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName) @@ -355,15 +366,16 @@ func (h *S3TablesHandler) authorizeMaintenanceBucket(r *http.Request, filerClien IdentityActions: getIdentityActions(r), DefaultAllow: h.defaultAllowFor(r), }) { - return "", NewAuthError(operation, principal, "not authorized to "+operation) + return "", time.Time{}, NewAuthError(operation, principal, "not authorized to "+operation) } - return bucketARN, nil + return bucketARN, bucketMetadata.CreatedAt, nil } // authorizeMaintenanceTable checks the caller may perform operation on the -// table, returning the table ARN. -func (h *S3TablesHandler) authorizeMaintenanceTable(r *http.Request, filerClient FilerClient, operation, bucketName, namespaceName, tableName string) (string, error) { +// table, returning the table ARN and the creation time of the table record the +// check read. +func (h *S3TablesHandler) authorizeMaintenanceTable(r *http.Request, filerClient FilerClient, operation, bucketName, namespaceName, tableName string) (string, time.Time, error) { tablePath := GetTablePath(bucketName, namespaceName, tableName) bucketPath := GetTableBucketPath(bucketName) @@ -391,7 +403,7 @@ func (h *S3TablesHandler) authorizeMaintenanceTable(r *http.Request, filerClient return err }) if err != nil { - return "", err + return "", time.Time{}, err } tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName) @@ -409,10 +421,10 @@ func (h *S3TablesHandler) authorizeMaintenanceTable(r *http.Request, filerClient } if deniedByPolicy(operation, principal, tablePolicy, tableARN, ctx) || !CheckPermissionWithContext(operation, principal, metadata.OwnerAccountID, bucketPolicy, tableARN, ctx) { - return "", NewAuthError(operation, principal, "not authorized to "+operation) + return "", time.Time{}, NewAuthError(operation, principal, "not authorized to "+operation) } - return tableARN, nil + return tableARN, metadata.CreatedAt, nil } // readPolicy returns the policy stored on path, or an empty string when the diff --git a/weed/s3api/s3tables/handler_namespace.go b/weed/s3api/s3tables/handler_namespace.go index a4690f491a..f693ce833e 100644 --- a/weed/s3api/s3tables/handler_namespace.go +++ b/weed/s3api/s3tables/handler_namespace.go @@ -151,6 +151,11 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R return err } + if otherTableBucket(r.Context(), bucketMetadata.CreatedAt) { + h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) + return filer_pb.ErrNotFound + } + bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName) principal := h.getAccountID(r) identityActions := getIdentityActions(r) @@ -170,20 +175,50 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R } namespacePath := GetNamespacePath(bucketName, namespaceName) + // The authorization above holds for this bucket only. One deleted and + // created again under the same name since then is another bucket, and is + // treated as gone by both checks below. + isAuthorizedBucket := isTableBucketCreatedAt(bucketMetadata.CreatedAt) - // Check if namespace already exists + // A delete that has claimed the bucket is scanning it for namespaces, so + // refuse before writing anything. Checked after the authorization above, so + // denied callers never see the delete. err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - _, err := h.readNamespaceMetadataAttr(r.Context(), client, namespacePath) + return h.checkParentOpen(r.Context(), client, bucketPath, isAuthorizedBucket) + }) + if h.writeBucketClosedError(w, err, bucketName) { return err + } + if err != nil { + h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table bucket: %v", err)) + return err + } + + // Check if namespace already exists. One still marked unfinished does not: + // its create either confirmed it or lost it, and the mark left behind would + // wedge the name, so this create takes it over the way a table's does. + exists := false + err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + entry, err := h.lookupEntry(r.Context(), client, namespacePath) + if errors.Is(err, filer_pb.ErrNotFound) { + return nil + } + if err != nil { + return err + } + _, hasMetadata := entry.Extended[ExtendedKeyMetadata] + exists = hasMetadata && !isUnfinishedCreate(entry.Extended) + return nil }) - if err == nil { - h.writeError(w, http.StatusConflict, ErrCodeNamespaceAlreadyExists, fmt.Sprintf("namespace %s already exists", namespaceName)) - return fmt.Errorf("namespace already exists") - } else if !errors.Is(err, filer_pb.ErrNotFound) { + if err != nil { h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check namespace: %v", err)) return err } + if exists { + h.writeError(w, http.StatusConflict, ErrCodeNamespaceAlreadyExists, fmt.Sprintf("namespace %s already exists", namespaceName)) + return fmt.Errorf("namespace already exists") + } // Create the namespace with bucket owner to maintain consistency // (authorization above ensures the caller has permission to create in this bucket) @@ -201,7 +236,13 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R return fmt.Errorf("failed to marshal metadata: %w", err) } - attrs := map[string][]byte{ExtendedKeyMetadata: metadataBytes} + // The unfinished mark keeps a table or view from being created in the + // namespace until the bucket check below has passed: one created before it + // would be left under a namespace that check then backs out. + attrs := map[string][]byte{ + ExtendedKeyMetadata: metadataBytes, + ExtendedKeyDeleting: []byte(createClaim), + } var taken error err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { err := h.createDirectoryExclusive(r.Context(), client, namespacePath, attrs) @@ -215,7 +256,19 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R return nil } } - return err + if err != nil { + return err + } + // A delete may have claimed the bucket since the check above, after it + // scanned for namespaces and before this write. Confirm the bucket once + // more now that the namespace is there: a delete claiming it later scans + // the namespace, marked or not, and one that claimed it already makes + // this back out what it wrote. The mark comes off only after that check + // and the bucket is not checked again: a table created in the window + // between the two would be left under a namespace the second check backs out. + // A bucket recreated since the authorization fails the check as gone, so + // the namespace is backed out of it rather than left with this owner. + return h.confirmParentOnce(r.Context(), client, bucketPath, isAuthorizedBucket, namespacePath, attrs, namespacePath) }) if err == nil && taken != nil { err = taken @@ -228,6 +281,9 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R h.writeError(w, http.StatusConflict, ErrCodeConflict, "namespace changed during create, retry the request") return err } + if h.writeBucketClosedError(w, err, bucketName) { + return err + } if err != nil { h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to create namespace") return err @@ -243,6 +299,25 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R return nil } +// writeBucketClosedError answers a create refused because its bucket is being +// created, being deleted, or was deleted, and reports whether err was one of +// those. +func (h *S3TablesHandler) writeBucketClosedError(w http.ResponseWriter, err error, bucketName string) bool { + switch { + case errors.Is(err, errParentCreating): + h.writeError(w, http.StatusConflict, ErrCodeConflict, fmt.Sprintf("table bucket %s is still being created, retry the request", bucketName)) + case errors.Is(err, errParentDeleting): + h.writeError(w, http.StatusConflict, ErrCodeConflict, fmt.Sprintf("table bucket %s is being deleted", bucketName)) + case errors.Is(err, errParentGone): + h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) + case errors.Is(err, errBackOutFailed): + h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("table bucket %s was deleted during the request and the new namespace may remain", bucketName)) + default: + return false + } + return true +} + // handleGetNamespace gets details of a namespace func (h *S3TablesHandler) handleGetNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error { var req GetNamespaceRequest @@ -425,6 +500,11 @@ func (h *S3TablesHandler) handleUpdateNamespace(w http.ResponseWriter, r *http.R err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, namespacePath, func(extended map[string][]byte) error { + // The write carries the authorized record's owner and creation + // time, so it must not land on a namespace created again since. + if !isNamespaceCreatedAt(metadata.CreatedAt)(&filer_pb.Entry{Extended: extended}) { + return fmt.Errorf("%w: %s holds another namespace", ErrConcurrentUpdate, namespacePath) + } extended[ExtendedKeyMetadata] = metadataBytes return nil }) @@ -580,9 +660,10 @@ func (h *S3TablesHandler) handleListNamespaces(w http.ResponseWriter, r *http.Re continue } - // Read metadata from extended attribute + // Read metadata from extended attribute. A namespace its create has + // not confirmed is not one yet. data, ok := entry.Entry.Extended[ExtendedKeyMetadata] - if !ok { + if !ok || isUnfinishedCreate(entry.Entry.Extended) { continue } @@ -676,6 +757,17 @@ func (h *S3TablesHandler) handleDeleteNamespace(w http.ResponseWriter, r *http.R if err := json.Unmarshal(data, &metadata); err != nil { return fmt.Errorf("failed to unmarshal metadata: %w", err) } + // Checked after the namespace read: a bucket created again holds only + // namespaces of its own, and the claim below is bound to this one. + if createdAt, bound := boundTableBucket(r.Context()); bound { + bucket, err := h.lookupEntry(r.Context(), client, bucketPath) + if err != nil { + return err + } + if !isTableBucketCreatedAt(createdAt)(bucket) { + return filer_pb.ErrNotFound + } + } // Fetch bucket policy if it exists policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy) @@ -718,11 +810,15 @@ func (h *S3TablesHandler) handleDeleteNamespace(w http.ResponseWriter, r *http.R } // Claim before scanning, so a create that writes after the scan sees it. + // A namespace whose create has not confirmed it is absent, and claiming it + // would replace the create's mark: a refused delete then lifts the claim and + // leaves the namespace looking confirmed before its create checked the bucket. + // One created again since it was authorized is absent too. var claim string var claimed map[string][]byte err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { var claimErr error - claim, claimed, claimErr = h.claimForDelete(r.Context(), client, namespacePath, isNamespaceEntry) + claim, claimed, claimErr = h.claimForDelete(r.Context(), client, namespacePath, isNamespaceCreatedAt(metadata.CreatedAt)) return claimErr }) if err != nil { diff --git a/weed/s3api/s3tables/handler_policy.go b/weed/s3api/s3tables/handler_policy.go index f6c89f6061..cf455df912 100644 --- a/weed/s3api/s3tables/handler_policy.go +++ b/weed/s3api/s3tables/handler_policy.go @@ -2,23 +2,26 @@ package s3tables import ( "bytes" + "context" "encoding/json" "errors" "fmt" "net/http" "strings" + "time" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" ) -// extractResourceOwnerAndBucket extracts ownership info and bucket name from resource metadata. -// This helper consolidates the repeated pattern used in handleTagResource, handleListTagsForResource, -// and handleUntagResource. +// extractResourceOwnerAndBucket extracts ownership info, bucket name and the +// creation time that tells the resource apart from one created again under its +// name. It consolidates the repeated pattern used in handleTagResource, +// handleListTagsForResource, and handleUntagResource. func (h *S3TablesHandler) extractResourceOwnerAndBucket( data []byte, resourcePath string, rType ResourceType, -) (ownerAccountID, bucketName string, err error) { +) (ownerAccountID, bucketName string, createdAt time.Time, err error) { // Extract bucket name from resource path (format: /buckets/{bucket}/... for both tables and buckets) parts := strings.Split(strings.Trim(resourcePath, "/"), "/") if len(parts) >= 2 { @@ -28,17 +31,76 @@ func (h *S3TablesHandler) extractResourceOwnerAndBucket( if rType == ResourceTypeTable { var meta tableMetadataInternal if err := json.Unmarshal(data, &meta); err != nil { - return "", "", err + return "", "", time.Time{}, err } - ownerAccountID = meta.OwnerAccountID + ownerAccountID, createdAt = meta.OwnerAccountID, meta.CreatedAt } else { var meta tableBucketMetadata if err := json.Unmarshal(data, &meta); err != nil { - return "", "", err + return "", "", time.Time{}, err } - ownerAccountID = meta.OwnerAccountID + ownerAccountID, createdAt = meta.OwnerAccountID, meta.CreatedAt } - return ownerAccountID, bucketName, nil + return ownerAccountID, bucketName, createdAt, nil +} + +type maxTagsKey struct{} + +var errTooManyTags = errors.New("too many tags") + +// WithMaxTags caps how many tags a TagResource run through Execute may leave +// on its resource. In-process only, like WithTableBucketCreatedAt. +func WithMaxTags(ctx context.Context, n int) context.Context { + return context.WithValue(ctx, maxTagsKey{}, n) +} + +// updateTags applies set and remove to raw, the tags the record holds at the +// write, so concurrent tag writes do not undo each other. +func updateTags(raw []byte, set map[string]string, remove []string) (map[string]string, error) { + tags := make(map[string]string) + if len(raw) > 0 { + if err := json.Unmarshal(raw, &tags); err != nil { + return nil, err + } + } + for k, v := range set { + tags[k] = v + } + for _, k := range remove { + delete(tags, k) + } + return tags, nil +} + +func writeTags(extended map[string][]byte, key string, tags map[string]string) error { + data, err := json.Marshal(tags) + if err != nil { + return err + } + extended[key] = data + return nil +} + +// requireBoundTableBucket answers not found when ctx binds the request to a +// table bucket other than the one now named bucketName. +func (h *S3TablesHandler) requireBoundTableBucket(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketName string) error { + if _, bound := ctx.Value(tableBucketCreatedAtKey{}).(time.Time); !bound { + return nil + } + data, err := h.getExtendedAttribute(ctx, client, GetTableBucketPath(bucketName), ExtendedKeyMetadata) + if errors.Is(err, ErrAttributeNotFound) { + return filer_pb.ErrNotFound + } else if err != nil { + return err + } + var meta tableBucketMetadata + if err := json.Unmarshal(data, &meta); err != nil { + return err + } + if otherTableBucket(ctx, meta.CreatedAt) { + return filer_pb.ErrNotFound + } + return nil } // handlePutTableBucketPolicy puts a policy on a table bucket @@ -122,6 +184,9 @@ func (h *S3TablesHandler) handlePutTableBucketPolicy(w http.ResponseWriter, r *h // landing mid-request sends the caller back through authorization. err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateEntryExtended(r.Context(), client, bucketPath, func(extended map[string][]byte) error { + if err := requireTableBucketCreatedAt(extended, bucketPath, bucketMetadata.CreatedAt); err != nil { + return err + } if !bytes.Equal(extended[ExtendedKeyPolicy], []byte(bucketPolicy)) { return fmt.Errorf("%w: %s", ErrConcurrentUpdate, ExtendedKeyPolicy) } @@ -133,6 +198,8 @@ func (h *S3TablesHandler) handlePutTableBucketPolicy(w http.ResponseWriter, r *h if err != nil { if errors.Is(err, ErrConcurrentUpdate) { h.writeError(w, http.StatusConflict, ErrCodeConflict, "table bucket policy changed concurrently, retry the request") + } else if errors.Is(err, filer_pb.ErrNotFound) { + h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) } else { h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to put table bucket policy") } @@ -288,11 +355,26 @@ func (h *S3TablesHandler) handleDeleteTableBucketPolicy(w http.ResponseWriter, r } err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - return h.deleteExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy) + return h.mutateEntryExtended(r.Context(), client, bucketPath, func(extended map[string][]byte) error { + if err := requireTableBucketCreatedAt(extended, bucketPath, bucketMetadata.CreatedAt); err != nil { + return err + } + if !bytes.Equal(extended[ExtendedKeyPolicy], []byte(bucketPolicy)) { + return fmt.Errorf("%w: %s", ErrConcurrentUpdate, ExtendedKeyPolicy) + } + delete(extended, ExtendedKeyPolicy) + return nil + }) }) - if err != nil && !errors.Is(err, ErrAttributeNotFound) { - h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table bucket policy") + if err != nil { + if errors.Is(err, ErrConcurrentUpdate) { + h.writeError(w, http.StatusConflict, ErrCodeConflict, "table bucket policy changed concurrently, retry the request") + } else if errors.Is(err, filer_pb.ErrNotFound) { + h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) + } else { + h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table bucket policy") + } return err } @@ -401,6 +483,9 @@ func (h *S3TablesHandler) handlePutTablePolicy(w http.ResponseWriter, r *http.Re err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, tablePath, func(extended map[string][]byte) error { + if err := requireTableCreatedAt(extended, tablePath, metadata.CreatedAt); err != nil { + return err + } if !bytes.Equal(extended[ExtendedKeyPolicy], []byte(tablePolicy)) { return fmt.Errorf("%w: %s", ErrConcurrentUpdate, ExtendedKeyPolicy) } @@ -617,15 +702,26 @@ func (h *S3TablesHandler) handleDeleteTablePolicy(w http.ResponseWriter, r *http return NewAuthError("DeleteTablePolicy", principal, "not authorized to delete table policy") } + // The removal is conditional on the table and the policy this request was + // authorized against, as PutTablePolicy's write is. err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - return h.removeExtendedAttributesIf(r.Context(), client, tablePath, - map[string][]byte{ExtendedKeyPolicy: []byte(tablePolicy)}, - ExtendedKeyPolicy) + return h.mutateCatalogRecord(r.Context(), client, tablePath, func(extended map[string][]byte) error { + if err := requireTableCreatedAt(extended, tablePath, metadata.CreatedAt); err != nil { + return err + } + if !bytes.Equal(extended[ExtendedKeyPolicy], []byte(tablePolicy)) { + return fmt.Errorf("%w: %s", ErrConcurrentUpdate, ExtendedKeyPolicy) + } + delete(extended, ExtendedKeyPolicy) + return nil + }) }) if err != nil && !errors.Is(err, ErrAttributeNotFound) { if errors.Is(err, ErrConcurrentUpdate) { h.writeError(w, http.StatusConflict, ErrCodeConflict, "table policy changed concurrently, retry the request") + } else if errors.Is(err, filer_pb.ErrNotFound) { + h.writeError(w, http.StatusNotFound, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName)) } else { h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table policy") } @@ -666,6 +762,7 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque var bucketPolicy, tablePolicy string var bucketTags map[string]string var tableNamespace, tableName string + var createdAt time.Time requestTagKeys := mapKeys(req.Tags) err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { // Read metadata for ownership check @@ -677,10 +774,11 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque return err } - ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType) + ownerAccountID, bucketName, resourceCreatedAt, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType) if err != nil { return err } + createdAt = resourceCreatedAt // Fetch bucket policy if we have a bucket name if bucketName != "" { @@ -713,6 +811,10 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque return err } + // Checked after the reads, so they all came from the bound bucket. + if err := h.requireBoundTableBucket(r.Context(), client, bucketName); err != nil { + return err + } var resourceARN string if rType == ResourceTypeTable { resourceARN = h.generateTableARN(ownerAccountID, bucketName, tableNamespace+"/"+tableName) @@ -756,21 +858,24 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque return err } - // Merge new tags - for k, v := range req.Tags { - existingTags[k] = v - } - - // Write merged tags - tagsBytes, err := json.Marshal(existingTags) - if err != nil { - h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal tags") - return fmt.Errorf("failed to marshal tags: %w", err) + sameResource := requireTableCreatedAt + if rType != ResourceTypeTable { + sameResource = requireTableBucketCreatedAt } + maxTags, limited := r.Context().Value(maxTagsKey{}).(int) err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, resourcePath, func(extended map[string][]byte) error { - extended[extendedKey] = tagsBytes - return nil + if err := sameResource(extended, resourcePath, createdAt); err != nil { + return err + } + tags, err := updateTags(extended[extendedKey], req.Tags, nil) + if err != nil { + return err + } + if limited && len(tags) > maxTags { + return errTooManyTags + } + return writeTags(extended, extendedKey, tags) }) }) @@ -783,6 +888,11 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque h.writeError(w, http.StatusNotFound, errorCode, "resource not found") return err } + if errors.Is(err, errTooManyTags) { + h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, + fmt.Sprintf("a resource can have at most %d tags", maxTags)) + return err + } if errors.Is(err, ErrConcurrentUpdate) { h.writeError(w, http.StatusConflict, ErrCodeConflict, "resource tags changed concurrently, retry the request") return err @@ -829,7 +939,7 @@ func (h *S3TablesHandler) handleListTagsForResource(w http.ResponseWriter, r *ht return err } - ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType) + ownerAccountID, bucketName, _, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType) if err != nil { return err } @@ -864,6 +974,10 @@ func (h *S3TablesHandler) handleListTagsForResource(w http.ResponseWriter, r *ht return err } + // Checked after the reads, so they all came from the bound bucket. + if err := h.requireBoundTableBucket(r.Context(), client, bucketName); err != nil { + return err + } var resourceARN string if rType == ResourceTypeTable { resourceARN = h.generateTableARN(ownerAccountID, bucketName, tableNamespace+"/"+tableName) @@ -942,6 +1056,7 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req var bucketPolicy, tablePolicy string var bucketTags map[string]string var tableNamespace, tableName string + var createdAt time.Time err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { // Read metadata for ownership check data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMetadata) @@ -952,10 +1067,11 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req return err } - ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType) + ownerAccountID, bucketName, resourceCreatedAt, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType) if err != nil { return err } + createdAt = resourceCreatedAt // Fetch bucket policy if we have a bucket name if bucketName != "" { @@ -987,6 +1103,10 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req return err } + // Checked after the reads, so they all came from the bound bucket. + if err := h.requireBoundTableBucket(r.Context(), client, bucketName); err != nil { + return err + } var resourceARN string if rType == ResourceTypeTable { resourceARN = h.generateTableARN(ownerAccountID, bucketName, tableNamespace+"/"+tableName) @@ -1029,21 +1149,20 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req return err } - // Remove specified tags - for _, key := range req.TagKeys { - delete(tags, key) - } - - // Write updated tags - tagsBytes, err := json.Marshal(tags) - if err != nil { - h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal tags") - return fmt.Errorf("failed to marshal tags: %w", err) + sameResource := requireTableCreatedAt + if rType != ResourceTypeTable { + sameResource = requireTableBucketCreatedAt } err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, resourcePath, func(extended map[string][]byte) error { - extended[extendedKey] = tagsBytes - return nil + if err := sameResource(extended, resourcePath, createdAt); err != nil { + return err + } + tags, err := updateTags(extended[extendedKey], nil, req.TagKeys) + if err != nil { + return err + } + return writeTags(extended, extendedKey, tags) }) }) diff --git a/weed/s3api/s3tables/handler_principal_test.go b/weed/s3api/s3tables/handler_principal_test.go index 93014a78c3..eed0eb57ba 100644 --- a/weed/s3api/s3tables/handler_principal_test.go +++ b/weed/s3api/s3tables/handler_principal_test.go @@ -30,13 +30,28 @@ func TestPrincipalForMirrorsGetAccountID(t *testing.T) { m := NewManager() m.SetAccountID("000000000000") - assert.Equal(t, "000000000000", m.PrincipalFor("")) - assert.Equal(t, "alice", m.PrincipalFor("alice")) - assert.Equal(t, s3_constants.AccountAdminId, m.PrincipalFor(s3_constants.AccountAdminId), + named := func(name string) *http.Request { + req := httptest.NewRequest(http.MethodGet, "/", nil) + return req.WithContext(s3_constants.SetIdentityNameInContext(req.Context(), name)) + } + withIdentity := func(id *testIdentity) *http.Request { + req := named(id.Name) + return req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id)) + } + + assert.Equal(t, "000000000000", m.PrincipalFor(nil)) + assert.Equal(t, "000000000000", m.PrincipalFor(named(""))) + assert.Equal(t, "alice", m.PrincipalFor(named("alice"))) + assert.Equal(t, s3_constants.AccountAdminId, m.PrincipalFor(named(s3_constants.AccountAdminId)), "the name branch of getAccountID keeps the admin name") - assert.Equal(t, "000000000000", m.PrincipalFor(" ")) - assert.Equal(t, "bob", m.PrincipalFor("arn:aws:iam::123456789012:user/bob"), + assert.Equal(t, "000000000000", m.PrincipalFor(named(" "))) + assert.Equal(t, "bob", m.PrincipalFor(named("arn:aws:iam::123456789012:user/bob")), "ARN identities collapse to the trailing segment like getAccountID") + assert.Equal(t, "team-acct", m.PrincipalFor(withIdentity(&testIdentity{Name: "carol", Account: &testIdentityAccount{Id: "team-acct"}})), + "an identity with an account is its account, not its name") + assert.Equal(t, "uid-alice", m.PrincipalFor(withIdentity(&testIdentity{Name: "alice-sub", Account: &testIdentityAccount{Id: "alice-sub"}, + Claims: map[string]interface{}{"aws:userid": "uid-alice"}})), + "a federated identity is its issuer-scoped user id") } func TestCheckPermissionDenyPolicyBindsNonAdmin(t *testing.T) { diff --git a/weed/s3api/s3tables/handler_table.go b/weed/s3api/s3tables/handler_table.go index 8e6ce93c96..6db8fab282 100644 --- a/weed/s3api/s3tables/handler_table.go +++ b/weed/s3api/s3tables/handler_table.go @@ -166,7 +166,7 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque if err := h.claimCatalogEntry(r.Context(), client, tablePath, attrs); err != nil { return err } - return h.confirmParent(r.Context(), client, GetNamespacePath(bucketName, namespaceName), isNamespaceEntry, tablePath, attrs, dataPath, tablePath) + return h.confirmParent(r.Context(), client, GetNamespacePath(bucketName, namespaceName), isNamespaceCreatedAt(target.namespaceCreatedAt), tablePath, attrs, dataPath, tablePath) }) if err != nil { @@ -188,8 +188,9 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName) resp := &CreateTableResponse{ - TableARN: tableARN, - VersionToken: versionToken, + TableARN: tableARN, + VersionToken: versionToken, + MetadataLocation: metadata.MetadataLocation, } h.writeJSON(w, http.StatusOK, resp) @@ -312,7 +313,7 @@ func (h *S3TablesHandler) handleRegisterTable(w http.ResponseWriter, r *http.Req if err := h.claimCatalogEntry(r.Context(), client, tablePath, attrs); err != nil { return err } - return h.confirmParent(r.Context(), client, GetNamespacePath(bucketName, namespaceName), isNamespaceEntry, tablePath, attrs, tablePath) + return h.confirmParent(r.Context(), client, GetNamespacePath(bucketName, namespaceName), isNamespaceCreatedAt(target.namespaceCreatedAt), tablePath, attrs, tablePath) }) if err != nil { if errors.Is(err, ErrNameTaken) { @@ -1008,6 +1009,10 @@ func (h *S3TablesHandler) handleDeleteTable(w http.ResponseWriter, r *http.Reque } return err } + if otherTableBucket(r.Context(), bucketMetadata.CreatedAt) { + h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) + return filer_pb.ErrNotFound + } tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName) bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName) @@ -1635,7 +1640,10 @@ func (h *S3TablesHandler) renameCatalogEntry(w http.ResponseWriter, r *http.Requ h.writeError(w, http.StatusConflict, kind.existsCode, fmt.Sprintf("%s %s already exists", kind.noun, destName)) return ErrTableAlreadyExists } - if err := h.refuseClaimedNamespace(w, r, filerClient, destNamespacePath, destNamespace); err != nil { + // The destination is bound to the namespace the create check ran against: + // one deleted and created again under its name reads as gone. + isDestNamespace := isNamespaceCreatedAt(destNamespaceMetadata.CreatedAt) + if err := h.refuseClaimedNamespace(w, r, filerClient, destNamespacePath, destNamespace, isDestNamespace); err != nil { return err } @@ -1717,7 +1725,7 @@ func (h *S3TablesHandler) renameCatalogEntry(w http.ResponseWriter, r *http.Requ h.removeLeftoverDirectory(ctx, client, destPath) return releaseClaim(err) } - destParent, err := h.openParent(r.Context(), client, destNamespacePath, isNamespaceEntry) + destParent, err := h.openParent(r.Context(), client, destNamespacePath, isDestNamespace) if err != nil { return backOutDest(err) } @@ -1743,7 +1751,7 @@ func (h *S3TablesHandler) renameCatalogEntry(w http.ResponseWriter, r *http.Requ destConfirmed := false if err = h.removeExtendedAttributesIf(commitCtx, client, destPath, written, ExtendedKeyDeleting); err == nil { destConfirmed = true - err = h.parentStillOpen(commitCtx, client, destNamespacePath, isNamespaceEntry, destParent) + err = h.parentStillOpen(commitCtx, client, destNamespacePath, isDestNamespace, destParent) } if err != nil { if destConfirmed { @@ -2115,13 +2123,15 @@ func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Reque } // createTableTarget is the namespace a create or register resolved to, once the -// caller has been authorized to put a table there. +// caller has been authorized to put a table there. namespaceCreatedAt names the +// namespace record the authorization read, so the write lands in no other. type createTableTarget struct { - bucketName string - namespaceName string - tableName string - bucketFormat string - ownerAccountID string + bucketName string + namespaceName string + tableName string + bucketFormat string + ownerAccountID string + namespaceCreatedAt time.Time } // authorizeCreateTable validates the names a create names and checks the caller @@ -2227,6 +2237,10 @@ func (h *S3TablesHandler) authorizeCreateTable(w http.ResponseWriter, r *http.Re h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to fetch policies: %v", err)) return nil, err } + if otherTableBucket(r.Context(), bucketMetadata.CreatedAt) { + h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName)) + return nil, filer_pb.ErrNotFound + } if !namespaceExists { // The implicit namespace has no record yet: authorize it as the @@ -2255,24 +2269,28 @@ func (h *S3TablesHandler) authorizeCreateTable(w http.ResponseWriter, r *http.Re } if namespaceExists { - if err := h.refuseClaimedNamespace(w, r, filerClient, namespacePath, namespaceName); err != nil { + if err := h.refuseClaimedNamespace(w, r, filerClient, namespacePath, namespaceName, isNamespaceCreatedAt(nsMetadata.CreatedAt)); err != nil { return nil, err } } return &createTableTarget{ - bucketName: bucketName, - namespaceName: namespaceName, - tableName: tableName, - bucketFormat: bucketMetadata.Format, - ownerAccountID: nsMetadata.OwnerAccountID, + bucketName: bucketName, + namespaceName: namespaceName, + tableName: tableName, + bucketFormat: bucketMetadata.Format, + ownerAccountID: nsMetadata.OwnerAccountID, + namespaceCreatedAt: nsMetadata.CreatedAt, }, nil } // writeNamespaceClosedError answers a create refused because its namespace is -// being deleted or was deleted, and reports whether err was one of those. +// being created, being deleted, or was deleted, and reports whether err was one +// of those. func (h *S3TablesHandler) writeNamespaceClosedError(w http.ResponseWriter, err error, namespaceName string) bool { switch { + case errors.Is(err, errParentCreating): + h.writeError(w, http.StatusConflict, ErrCodeConflict, fmt.Sprintf("namespace %s is still being created, retry the request", namespaceName)) case errors.Is(err, errParentDeleting): h.writeError(w, http.StatusConflict, ErrCodeConflict, fmt.Sprintf("namespace %s is being deleted", namespaceName)) case errors.Is(err, errParentGone), errors.Is(err, filer_pb.ErrNotFound): @@ -2285,13 +2303,16 @@ func (h *S3TablesHandler) writeNamespaceClosedError(w http.ResponseWriter, err e return true } -// refuseClaimedNamespace answers 409 when a delete has claimed the namespace. -// Call it after the authorization check, so denied callers never see the delete. -func (h *S3TablesHandler) refuseClaimedNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient, namespacePath, namespaceName string) error { +// refuseClaimedNamespace refuses a request before it writes anything when the +// namespace is not the open record the caller was authorized against: 409 when a +// delete has claimed it, 404 when it is gone, was replaced by another namespace, +// or its create is not confirmed yet. isNamespace accepts that record. Call it +// after the authorization check, so denied callers never see the delete. +func (h *S3TablesHandler) refuseClaimedNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient, namespacePath, namespaceName string, isNamespace func(*filer_pb.Entry) bool) error { err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - return h.checkParentOpen(r.Context(), client, namespacePath, isNamespaceEntry) + return h.checkParentOpen(r.Context(), client, namespacePath, isNamespace) }) - if errors.Is(err, errParentDeleting) { + if errors.Is(err, errParentDeleting) || errors.Is(err, errParentCreating) || errors.Is(err, errParentGone) { h.writeNamespaceClosedError(w, err, namespaceName) return err } diff --git a/weed/s3api/s3tables/handler_update_conflict_test.go b/weed/s3api/s3tables/handler_update_conflict_test.go index 0117f736dc..5b4066b772 100644 --- a/weed/s3api/s3tables/handler_update_conflict_test.go +++ b/weed/s3api/s3tables/handler_update_conflict_test.go @@ -67,3 +67,23 @@ func TestUntagResourceClaimedRecordConflicts(t *testing.T) { assert.Equal(t, http.StatusConflict, status, "%+v", resp) assert.Equal(t, ErrCodeConflict, resp.Type) } + +// DeleteTablePolicy writes like every other mutation of a table record, so a +// record another operation claimed refuses it with 409 instead of losing the +// policy under the claim's owner. +func TestDeleteTablePolicyClaimedRecordConflicts(t *testing.T) { + fs, m := startTableT(t, map[string][]byte{ExtendedKeyPolicy: []byte(bindTestPolicy)}) + nsPath := GetNamespacePath(renameTestBucket, "ns") + claimRecord(t, fs, nsPath, "t") + + status, resp := runHandler(t, m, fs.Client, "DeleteTablePolicy", &DeleteTablePolicyRequest{ + TableBucketARN: mustBucketARN(t), + Namespace: []string{"ns"}, + Name: "t", + }) + assert.Equal(t, http.StatusConflict, status, "%+v", resp) + assert.Equal(t, ErrCodeConflict, resp.Type, "%+v", resp) + extended := fs.Get(nsPath, "t").Extended + assert.Equal(t, []byte(bindTestPolicy), extended[ExtendedKeyPolicy], "the refused delete keeps the policy") + assert.Equal(t, []byte("delete"), extended[ExtendedKeyDeleting], "and leaves the claim to its owner") +} diff --git a/weed/s3api/s3tables/handler_view.go b/weed/s3api/s3tables/handler_view.go index cb63a3b97c..e8f6dadb61 100644 --- a/weed/s3api/s3tables/handler_view.go +++ b/weed/s3api/s3tables/handler_view.go @@ -1,6 +1,7 @@ package s3tables import ( + "bytes" "context" "encoding/json" "errors" @@ -85,7 +86,9 @@ func (h *S3TablesHandler) handleCreateView(w http.ResponseWriter, r *http.Reques h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create view in this namespace") return ErrAccessDenied } - if err := h.refuseClaimedNamespace(w, r, filerClient, namespacePath, namespaceName); err != nil { + // The view is bound to the namespace record it was authorized against. + isNamespace := isNamespaceCreatedAt(namespaceMetadata.CreatedAt) + if err := h.refuseClaimedNamespace(w, r, filerClient, namespacePath, namespaceName, isNamespace); err != nil { return err } @@ -169,7 +172,7 @@ func (h *S3TablesHandler) handleCreateView(w http.ResponseWriter, r *http.Reques if err := h.claimCatalogEntry(r.Context(), client, viewPath, attrs); err != nil { return err } - return h.confirmParent(r.Context(), client, namespacePath, isNamespaceEntry, viewPath, attrs, viewPath) + return h.confirmParent(r.Context(), client, namespacePath, isNamespace, viewPath, attrs, viewPath) }) if err != nil { if errors.Is(err, ErrNameTaken) { @@ -432,14 +435,17 @@ func (h *S3TablesHandler) listViewsInNamespace(r *http.Request, client filer_pb. } // updateViewTarget is the view state an update was authorized against: the -// metadata the version-token check reads and the fields the response echoes. +// metadata the version-token check reads, the stored values the +// optimistic-concurrency check compares and the fields the response echoes. type updateViewTarget struct { - bucketName string - namespaceName string - viewName string - viewPath string - viewARN string - metadata tableMetadataInternal + bucketName string + namespaceName string + viewName string + viewPath string + viewARN string + metadata tableMetadataInternal + storedMetadata []byte + storedPolicy []byte } // authorizeUpdateView validates the names an update names, loads the view and @@ -460,6 +466,7 @@ func (h *S3TablesHandler) authorizeUpdateView(w http.ResponseWriter, r *http.Req viewPath := GetTablePath(bucketName, namespaceName, viewName) var metadata tableMetadataInternal + var storedMetadata []byte err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { entry, err := h.lookupEntry(r.Context(), client, viewPath) if err != nil { @@ -472,6 +479,7 @@ func (h *S3TablesHandler) authorizeUpdateView(w http.ResponseWriter, r *http.Req if !ok { return fmt.Errorf("%w: %w: %s", filer_pb.ErrNotFound, ErrAttributeNotFound, ExtendedKeyMetadata) } + storedMetadata = data return json.Unmarshal(data, &metadata) }) if err != nil { @@ -496,12 +504,14 @@ func (h *S3TablesHandler) authorizeUpdateView(w http.ResponseWriter, r *http.Req } return &updateViewTarget{ - bucketName: bucketName, - namespaceName: namespaceName, - viewName: viewName, - viewPath: viewPath, - viewARN: viewARN, - metadata: metadata, + bucketName: bucketName, + namespaceName: namespaceName, + viewName: viewName, + viewPath: viewPath, + viewARN: viewARN, + metadata: metadata, + storedMetadata: storedMetadata, + storedPolicy: []byte(viewPolicy), }, nil } @@ -558,8 +568,18 @@ func (h *S3TablesHandler) handleUpdateView(w http.ResponseWriter, r *http.Reques return err } + // Conditional on the record this request read and authorized against, as + // UpdateTable's write is: a concurrent update would otherwise be dropped, + // and a view deleted and created again under the name would take a write + // nobody authorized for it. err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { return h.mutateCatalogRecord(r.Context(), client, viewPath, func(extended map[string][]byte) error { + if !bytes.Equal(extended[ExtendedKeyMetadata], target.storedMetadata) { + return fmt.Errorf("%w: %s", ErrConcurrentUpdate, ExtendedKeyMetadata) + } + if !bytes.Equal(extended[ExtendedKeyPolicy], target.storedPolicy) { + return fmt.Errorf("%w: %s", ErrConcurrentUpdate, ExtendedKeyPolicy) + } extended[ExtendedKeyMetadata] = metadataBytes return nil }) diff --git a/weed/s3api/s3tables/manager.go b/weed/s3api/s3tables/manager.go index c051a5ecb9..0572d74a36 100644 --- a/weed/s3api/s3tables/manager.go +++ b/weed/s3api/s3tables/manager.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/http/httptest" + "time" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" @@ -46,30 +47,60 @@ func (m *Manager) SetTrusted(trusted bool) { m.handler.SetTrusted(trusted) } -// PrincipalFor maps a forwarded identity name onto the account the catalog -// would charge the request to: a synthesized request resolves its identity -// name through getAccountID's name branch — normalized like any principal — -// and an empty name falls back to the handler's account ID. -func (m *Manager) PrincipalFor(identity string) string { - if principal := normalizePrincipalID(identity); principal != "" { - return principal +// SetEncryptionChecker sets the handler's SSE-S3 availability predicate. +func (m *Manager) SetEncryptionChecker(fn func() bool) { + m.handler.SetEncryptionChecker(fn) +} + +// PrincipalFor resolves the caller of r the way the catalog's handlers do, +// so it matches the owner CreateTableBucket records. A nil request is the +// handler's account ID. +func (m *Manager) PrincipalFor(r *http.Request) string { + if r == nil { + return m.handler.accountID } - return m.handler.accountID + return m.handler.getAccountID(r) } // PolicyContext builds the evaluation context a Manager call's synthesized // request produces: the configured local-account predicate, the caller's own // identity actions so admin and action grants resolve on privileges rather // than the forwarded name, plus the open-by-default fallback only for a -// trusted caller or no resolved identity. -func (m *Manager) PolicyContext(r *http.Request) *PolicyContext { - ctx := &PolicyContext{IsLocalAccountID: m.handler.localAccountIDs, IdentityActions: getIdentityActions(r)} +// trusted caller or no resolved identity. A non-empty bucket lets an identity +// action scoped to that bucket apply, as the handlers do. +func (m *Manager) PolicyContext(r *http.Request, bucket string) *PolicyContext { + ctx := &PolicyContext{IsLocalAccountID: m.handler.localAccountIDs, IdentityActions: getIdentityActions(r), + TrustedIssuers: m.handler.trustedIssuers(r), TableBucketName: bucket} if m.handler.trusted || (m.handler.defaultAllow && s3_constants.GetIdentityNameFromContext(r) == "") { ctx.DefaultAllow = true } return ctx } +type tableBucketCreatedAtKey struct{} + +// WithTableBucketCreatedAt binds a DeleteTableBucket, CreateNamespace, +// DeleteNamespace, CreateTable, DeleteTable or tag operation run through +// Execute to the bucket created at createdAt, so a caller that authorized the +// bucket itself never writes into one created again under its name. +// In-process only: no wire request can carry it. +func WithTableBucketCreatedAt(ctx context.Context, createdAt time.Time) context.Context { + return context.WithValue(ctx, tableBucketCreatedAtKey{}, createdAt) +} + +// boundTableBucket returns the bucket creation time ctx binds the request to. +func boundTableBucket(ctx context.Context) (time.Time, bool) { + createdAt, ok := ctx.Value(tableBucketCreatedAtKey{}).(time.Time) + return createdAt, ok +} + +// otherTableBucket reports whether ctx binds the request to a bucket other +// than the one created at createdAt. +func otherTableBucket(ctx context.Context, createdAt time.Time) bool { + want, ok := boundTableBucket(ctx) + return ok && !createdAt.Equal(want) +} + // Execute runs an S3 Tables operation and decodes the response into resp (if provided). func (m *Manager) Execute(ctx context.Context, filerClient FilerClient, operation string, req interface{}, resp interface{}, identity string) error { body, err := json.Marshal(req) diff --git a/weed/s3api/s3tables/s3tablestest/memfiler.go b/weed/s3api/s3tables/s3tablestest/memfiler.go index 391a2f0331..292a7c94cd 100644 --- a/weed/s3api/s3tables/s3tablestest/memfiler.go +++ b/weed/s3api/s3tables/s3tablestest/memfiler.go @@ -204,12 +204,10 @@ func (f *MemFiler) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryReque } f.mu.Lock() defer f.mu.Unlock() - // The real filer refuses a non-recursive delete of a non-empty folder. - if child := path.Join(req.Directory, req.Name); !req.IsRecursive && len(f.entries[child]) > 0 { - return &filer_pb.DeleteEntryResponse{Error: "fail to delete non-empty folder: " + child}, nil - } // The real filer evaluates ExpectedExtended under the path lock and refuses - // the delete when the entry no longer matches what the caller captured. + // the delete when the entry no longer matches what the caller captured. It + // does so before anything else, so a directory that changed hands is a + // changed entry whether or not it also holds something. for key, expected := range req.ExpectedExtended { var actual []byte if d, ok := f.entries[req.Directory]; ok { @@ -221,6 +219,10 @@ func (f *MemFiler) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryReque return nil, status.Errorf(codes.FailedPrecondition, "extended attribute %q changed", key) } } + // The real filer then refuses a non-recursive delete of a non-empty folder. + if child := path.Join(req.Directory, req.Name); !req.IsRecursive && len(f.entries[child]) > 0 { + return &filer_pb.DeleteEntryResponse{Error: "fail to delete non-empty folder: " + child}, nil + } if d, ok := f.entries[req.Directory]; ok { delete(d, req.Name) } diff --git a/weed/s3api/s3tables/utils.go b/weed/s3api/s3tables/utils.go index f87e34e3fc..7837ed9cbc 100644 --- a/weed/s3api/s3tables/utils.go +++ b/weed/s3api/s3tables/utils.go @@ -3,6 +3,7 @@ package s3tables import ( "crypto/rand" "encoding/hex" + "encoding/json" "fmt" "net/url" "path" @@ -192,6 +193,24 @@ func ValidateEncryptionConfiguration(cfg map[string]interface{}, sseField string return nil } +// DeclaredBucketEncryption returns the sseAlgorithm a table bucket's record +// declares, for the S3 write path to apply, when the bucket's format leaves +// every file to its clients. An ICEBERG bucket gets "": its catalog reads the +// manifests clients upload straight from the filer, past S3 decryption. +func DeclaredBucketEncryption(entry *filer_pb.Entry) string { + if !IsTableBucketEntry(entry) { + return "" + } + var metadata tableBucketMetadata + if json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata) != nil || !IsCatalogOnlyFormat(metadata.Format) { + return "" + } + if algorithm, _ := metadata.EncryptionConfiguration["sseAlgorithm"].(string); algorithm == "AES256" { + return algorithm + } + return "" +} + func parseS3Location(location string) (bucket, tablePath string, err error) { if !strings.HasPrefix(location, "s3://") { return "", "", fmt.Errorf("unsupported location: %s", location) diff --git a/weed/s3api/s3vectors/auth_test.go b/weed/s3api/s3vectors/auth_test.go index a333265d0d..a7e1fb88b7 100644 --- a/weed/s3api/s3vectors/auth_test.go +++ b/weed/s3api/s3vectors/auth_test.go @@ -1,8 +1,11 @@ package s3vectors import ( + "errors" + "io" "net/http" "net/http/httptest" + "strings" "testing" "github.com/seaweedfs/seaweedfs/weed/s3api" @@ -11,10 +14,18 @@ import ( ) type fakeAuthenticator struct { - identity *s3api.Identity + identity *s3api.Identity + errCode s3err.ErrorCode + calls int + bodyLimit int64 } func (f *fakeAuthenticator) AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode) { + f.calls++ + f.bodyLimit = s3_constants.GetRequestBodyLimitFromContext(r, 0) + if f.errCode != s3err.ErrNone { + return "", nil, f.errCode + } return f.identity.Name, f.identity, s3err.ErrNone } @@ -35,3 +46,55 @@ func TestAuthKeepsIdentityObject(t *testing.T) { t.Fatalf("identity object missing from context: %v", got) } } + +func TestAuthAnswersOversizedBodyWithValidationException(t *testing.T) { + alice := &s3api.Identity{Name: "alice", Actions: []s3api.Action{"Admin"}} + for _, tc := range []struct { + name string + contentLength int64 + errCode s3err.ErrorCode + wantStatus int + wantCode string + wantAuthCalls int + }{ + {"declared length over the limit", maxRequestBodyBytes + 1, s3err.ErrNone, http.StatusBadRequest, "ValidationException", 0}, + {"streamed body over the limit", -1, s3err.ErrEntityTooLarge, http.StatusBadRequest, "ValidationException", 1}, + {"bad signature", 10, s3err.ErrSignatureDoesNotMatch, http.StatusForbidden, "AccessDenied", 1}, + {"declared length at the limit", maxRequestBodyBytes, s3err.ErrNone, http.StatusOK, "", 1}, + } { + t.Run(tc.name, func(t *testing.T) { + auth := &fakeAuthenticator{identity: alice, errCode: tc.errCode} + s := &Server{authenticator: auth} + req := httptest.NewRequest(http.MethodPost, "/PutVectors", strings.NewReader("{}")) + req.ContentLength = tc.contentLength + rec := httptest.NewRecorder() + s.Auth(func(w http.ResponseWriter, r *http.Request) {})(rec, req) + + if rec.Code != tc.wantStatus || auth.calls != tc.wantAuthCalls { + t.Fatalf("status %d after %d auth calls, want %d after %d: %s", rec.Code, auth.calls, tc.wantStatus, tc.wantAuthCalls, rec.Body) + } + if auth.calls > 0 && auth.bodyLimit != maxRequestBodyBytes { + t.Fatalf("signature check reads up to %d bytes, want the route's %d", auth.bodyLimit, maxRequestBodyBytes) + } + if tc.wantCode != "" && !strings.Contains(rec.Body.String(), `"code":"`+tc.wantCode+`"`) { + t.Fatalf("body %s, want code %s", rec.Body, tc.wantCode) + } + if tc.wantCode == "ValidationException" && !strings.Contains(rec.Body.String(), "request body exceeds 32 MiB") { + t.Fatalf("body %s does not name the limit", rec.Body) + } + }) + } +} + +func TestReadJSONRefusesBodyOverTheLimit(t *testing.T) { + var req PutVectorsRequest + over := httptest.NewRequest(http.MethodPost, "/PutVectors", io.MultiReader( + strings.NewReader(`{"indexName":"`), strings.NewReader(strings.Repeat("a", maxRequestBodyBytes)), strings.NewReader(`"}`))) + if err := readJSON(over, &req); !errors.Is(err, errRequestBodyTooLarge) { + t.Fatalf("readJSON over the limit = %v, want %v", err, errRequestBodyTooLarge) + } + within := httptest.NewRequest(http.MethodPost, "/PutVectors", strings.NewReader(`{"indexName":"idx"}`)) + if err := readJSON(within, &req); err != nil || req.IndexName != "idx" { + t.Fatalf("readJSON within the limit = %v, indexName %q", err, req.IndexName) + } +} diff --git a/weed/s3api/s3vectors/bind_bucket_test.go b/weed/s3api/s3vectors/bind_bucket_test.go new file mode 100644 index 0000000000..b70a0cba36 --- /dev/null +++ b/weed/s3api/s3vectors/bind_bucket_test.go @@ -0,0 +1,243 @@ +package s3vectors + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" +) + +// A vector bucket policy write authorizes against the bucket it read: when the +// bucket is deleted and created again by another account before the write's +// compare-and-swap reads the entry, the write must not land on the new bucket. +func TestVectorBucketPolicyWritesKeepToAuthorizedBucket(t *testing.T) { + const policy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3vectors:GetVectorBucket","Resource":"*"}]}` + cases := []struct { + name string + call func(s *Server) *httptest.ResponseRecorder + }{ + {"put", func(s *Server) *httptest.ResponseRecorder { + return postJSONAs(t, "", s.handlePutVectorBucketPolicy, &PutVectorBucketPolicyRequest{VectorBucketName: "vec-bucket", Policy: policy}) + }}, + {"delete", func(s *Server) *httptest.ResponseRecorder { + return postJSONAs(t, "", s.handleDeleteVectorBucketPolicy, &DeleteVectorBucketPolicyRequest{VectorBucketName: "vec-bucket"}) + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, newFakeVectors(0)) + if w := postJSONAs(t, "", s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}); w.Code != http.StatusOK { + t.Fatalf("create bucket: %d %s", w.Code, w.Body.String()) + } + if w := postJSONAs(t, "", s.handlePutVectorBucketPolicy, &PutVectorBucketPolicyRequest{VectorBucketName: "vec-bucket", Policy: policy}); w.Code != http.StatusOK { + t.Fatalf("seed policy: %d %s", w.Code, w.Body.String()) + } + + dir, name := splitEntryPath(s3tables.GetTableBucketPath("vec-bucket")) + lookups := 0 + fs.OnLookup = func(d, n string) { + if d == dir && n == name { + lookups++ + } + } + if w := tc.call(s); w.Code != http.StatusOK { + t.Fatalf("baseline %s: %d %s", tc.name, w.Code, w.Body.String()) + } + fs.OnLookup = nil + if w := postJSONAs(t, "", s.handlePutVectorBucketPolicy, &PutVectorBucketPolicyRequest{VectorBucketName: "vec-bucket", Policy: policy}); w.Code != http.StatusOK { + t.Fatalf("reseed policy: %d %s", w.Code, w.Body.String()) + } + // The last bucket lookup is the compare-and-swap read; replace the + // bucket with another account's right before it. + last, seen := lookups, 0 + fs.OnLookup = func(d, n string) { + if d != dir || n != name { + return + } + if seen++; seen != last { + return + } + entry := fs.Get(dir, name) + var meta map[string]interface{} + if err := json.Unmarshal(entry.Extended[s3tables.ExtendedKeyMetadata], &meta); err != nil { + t.Errorf("decode bucket metadata: %v", err) + return + } + meta["ownerAccountId"] = "bob" + meta["createdAt"] = time.Now().Add(time.Hour).UTC() + data, _ := json.Marshal(meta) + ext := make(map[string][]byte, len(entry.Extended)) + for k, v := range entry.Extended { + ext[k] = v + } + ext[s3tables.ExtendedKeyMetadata] = data + ext[vectorPolicyKey] = []byte("bob-policy") + fs.Put(dir, name, ext) + } + w := tc.call(s) + fs.OnLookup = nil + if w.Code != http.StatusConflict { + t.Fatalf("%s over a replaced bucket = %d %s, want 409", tc.name, w.Code, w.Body.String()) + } + if got := string(fs.Get(dir, name).Extended[vectorPolicyKey]); got != "bob-policy" { + t.Fatalf("%s rewrote the replacing bucket's policy: %q", tc.name, got) + } + }) + } +} + +// replaceBucket swaps the bucket's record for another account's bucket of the +// same name, keeping its contents. +func replaceBucket(t *testing.T, fs *s3tablestest.MemFiler, bucket string) { + dir, name := splitEntryPath(s3tables.GetTableBucketPath(bucket)) + entry := fs.Get(dir, name) + var meta map[string]interface{} + if err := json.Unmarshal(entry.Extended[s3tables.ExtendedKeyMetadata], &meta); err != nil { + t.Errorf("decode bucket metadata: %v", err) + return + } + meta["ownerAccountId"] = "bob" + meta["createdAt"] = time.Now().Add(time.Hour).UTC() + data, _ := json.Marshal(meta) + ext := make(map[string][]byte, len(entry.Extended)) + for k, v := range entry.Extended { + ext[k] = v + } + ext[s3tables.ExtendedKeyMetadata] = data + fs.Put(dir, name, ext) +} + +// replaceBucketAfterAuthorization replaces bucket right after the lookups +// authorizeVector makes, counted on a probe bucket of the same shape. +func replaceBucketAfterAuthorization(t *testing.T, fs *s3tablestest.MemFiler, s *Server, bucket string) { + if w := postJSONAs(t, "", s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-probe"}); w.Code != http.StatusOK { + t.Fatalf("create probe bucket: %d %s", w.Code, w.Body.String()) + } + probeDir, probeName := splitEntryPath(s3tables.GetTableBucketPath("vec-probe")) + authLookups := 0 + fs.OnLookup = func(d, n string) { + if d == probeDir && n == probeName { + authLookups++ + } + } + probe := httptest.NewRequest(http.MethodPost, "/", nil) + probe = probe.WithContext(contextWithIdentity(probe.Context(), "")) + if _, err := s.authorizeVector(probe.Context(), probe, "s3vectors:DeleteVectorBucket", "vec-probe", ""); err != nil { + t.Fatalf("authorize probe: %v", err) + } + + dir, name := splitEntryPath(s3tables.GetTableBucketPath(bucket)) + seen := 0 + fs.OnLookup = func(d, n string) { + if d == dir && n == name { + if seen++; seen == authLookups+1 { + replaceBucket(t, fs, bucket) + } + } + } +} + +// DeleteVectorBucket deletes only the bucket it authorized: one deleted and +// created again by another account in between survives. +func TestDeleteVectorBucketKeepsReplacingBucket(t *testing.T) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, newFakeVectors(0)) + if w := postJSONAs(t, "", s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}); w.Code != http.StatusOK { + t.Fatalf("create bucket: %d %s", w.Code, w.Body.String()) + } + replaceBucketAfterAuthorization(t, fs, s, "vec-bucket") + w := postJSONAs(t, "", s.handleDeleteVectorBucket, &DeleteVectorBucketRequest{VectorBucketName: "vec-bucket"}) + fs.OnLookup = nil + if w.Code != http.StatusNotFound { + t.Fatalf("delete over a replaced bucket = %d %s, want 404", w.Code, w.Body.String()) + } + dir, name := splitEntryPath(s3tables.GetTableBucketPath("vec-bucket")) + if fs.Get(dir, name) == nil { + t.Fatal("DeleteVectorBucket removed the bucket that replaced the authorized one") + } +} + +// CreateIndex and DeleteIndex write only into the bucket they authorized. +func TestIndexWritesKeepToAuthorizedBucket(t *testing.T) { + newServer := func(t *testing.T) (*s3tablestest.MemFiler, *Server) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, newFakeVectors(0)) + if w := postJSONAs(t, "", s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}); w.Code != http.StatusOK { + t.Fatalf("create bucket: %d %s", w.Code, w.Body.String()) + } + return fs, s + } + createIndex := func(s *Server) *httptest.ResponseRecorder { + return postJSONAs(t, "", s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "euclidean", + }) + } + indexDir, indexName := splitEntryPath(s3tables.GetTablePath("vec-bucket", indexNamespace, "idx")) + + t.Run("create", func(t *testing.T) { + fs, s := newServer(t) + replaceBucketAfterAuthorization(t, fs, s, "vec-bucket") + w := createIndex(s) + fs.OnLookup = nil + if w.Code != http.StatusNotFound { + t.Fatalf("create over a replaced bucket = %d %s, want 404", w.Code, w.Body.String()) + } + if fs.Get(indexDir, indexName) != nil { + t.Fatal("CreateIndex wrote an index into the bucket that replaced the authorized one") + } + }) + t.Run("delete", func(t *testing.T) { + fs, s := newServer(t) + if w := createIndex(s); w.Code != http.StatusOK { + t.Fatalf("create index: %d %s", w.Code, w.Body.String()) + } + replaceBucketAfterAuthorization(t, fs, s, "vec-bucket") + w := postJSONAs(t, "", s.handleDeleteIndex, &DeleteIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx"}) + fs.OnLookup = nil + if w.Code != http.StatusNotFound { + t.Fatalf("delete over a replaced bucket = %d %s, want 404", w.Code, w.Body.String()) + } + if fs.Get(indexDir, indexName) == nil { + t.Fatal("DeleteIndex removed an index of the bucket that replaced the authorized one") + } + }) +} + +// DeleteVectorBucket drops the empty index namespace only from the bucket it +// authorized: a bucket created again before that cleanup keeps its own. +func TestDeleteVectorBucketCleanupKeepsReplacingBucket(t *testing.T) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, newFakeVectors(0)) + if w := postJSONAs(t, "", s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}); w.Code != http.StatusOK { + t.Fatalf("create bucket: %d %s", w.Code, w.Body.String()) + } + if w := postJSONAs(t, "", s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "euclidean", + }); w.Code != http.StatusOK { + t.Fatalf("create index: %d %s", w.Code, w.Body.String()) + } + if w := postJSONAs(t, "", s.handleDeleteIndex, &DeleteIndexRequest{VectorBucketName: "vec-bucket", IndexName: "idx"}); w.Code != http.StatusOK { + t.Fatalf("delete index: %d %s", w.Code, w.Body.String()) + } + + bucketPath := s3tables.GetTableBucketPath("vec-bucket") + fs.OnLookup = func(d, n string) { + if d == bucketPath && n == indexNamespace { + fs.OnLookup = nil + replaceBucket(t, fs, "vec-bucket") + } + } + w := postJSONAs(t, "", s.handleDeleteVectorBucket, &DeleteVectorBucketRequest{VectorBucketName: "vec-bucket"}) + fs.OnLookup = nil + if w.Code == http.StatusOK { + t.Fatalf("delete over a replaced bucket succeeded: %s", w.Body.String()) + } + if fs.Get(bucketPath, indexNamespace) == nil { + t.Fatal("DeleteVectorBucket removed the index namespace of the bucket that replaced the authorized one") + } +} diff --git a/weed/s3api/s3vectors/distance_metric_test.go b/weed/s3api/s3vectors/distance_metric_test.go index e3a149778e..502216bb8c 100644 --- a/weed/s3api/s3vectors/distance_metric_test.go +++ b/weed/s3api/s3vectors/distance_metric_test.go @@ -3,6 +3,7 @@ package s3vectors import ( "context" "encoding/json" + "fmt" "net/http" "testing" @@ -86,3 +87,54 @@ func TestIndexFromRecordLowercasesMetric(t *testing.T) { } } } + +type distancesFake struct { + *fakeVectors + distances []float32 +} + +func (f distancesFake) QueryVectors(ctx context.Context, req *vector_pb.QueryVectorsRequest, opts ...grpc.CallOption) (*vector_pb.QueryVectorsResponse, error) { + resp := &vector_pb.QueryVectorsResponse{} + for i, d := range f.distances { + resp.Vectors = append(resp.Vectors, &vector_pb.QueryResult{Key: fmt.Sprint(i), Distance: d}) + } + return resp, nil +} + +// The engine returns Lance's squared L2 distance; a euclidean index reports +// its square root, the other metrics their engine value. +func TestQueryVectorsReportsEuclideanDistance(t *testing.T) { + for metric, want := range map[string][]float32{ + "euclidean": {5, 0.5, 0}, + "cosine": {25, 0.25, -1e-7}, + "dotproduct": {25, 0.25, -1e-7}, + } { + fs := s3tablestest.Start(t) + engine := distancesFake{newFakeVectors(0), []float32{25, 0.25, -1e-7}} + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, engine) + if w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}); w.Code != http.StatusOK { + t.Fatalf("create bucket: %d %s", w.Code, w.Body.String()) + } + if w := postJSON(t, s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 2, DistanceMetric: metric, + }); w.Code != http.StatusOK { + t.Fatalf("create index: %d %s", w.Code, w.Body.String()) + } + w := postJSON(t, s.handleQueryVectors, &QueryVectorsRequest{ + VectorBucketName: "vec-bucket", IndexName: "idx", + QueryVector: &QueryVector{Float32: []float32{1, 0}}, TopK: 3, ReturnDistance: true, + }) + var query QueryVectorsResponse + if err := json.Unmarshal(w.Body.Bytes(), &query); err != nil || len(query.Vectors) != len(want) { + t.Fatalf("%s QueryVectors: %d %s", metric, w.Code, w.Body.String()) + } + for i, v := range query.Vectors { + if v.Distance == nil { + t.Fatalf("%s hit %d has no distance", metric, i) + } + if *v.Distance != want[i] { + t.Fatalf("%s hit %d distance %v, want %v", metric, i, *v.Distance, want[i]) + } + } + } +} diff --git a/weed/s3api/s3vectors/execute.go b/weed/s3api/s3vectors/execute.go new file mode 100644 index 0000000000..9a2a2e6872 --- /dev/null +++ b/weed/s3api/s3vectors/execute.go @@ -0,0 +1,54 @@ +package s3vectors + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" +) + +// Error is an S3 Vectors error response, as Execute returns it. +type Error struct { + Code string `json:"code"` + Message string `json:"message"` +} + +func (e *Error) Error() string { return e.Code + ": " + e.Message } + +// Execute runs one operation as identity for trusted local tooling such as +// the admin console, like s3tables.Manager.Execute: nothing is signed or +// authenticated, and identity's grants decide what the operation may do. +// Operations that reach the vector engine need the server's VectorServiceClient. +func (s *Server) Execute(ctx context.Context, operation string, req, resp interface{}, identity string) error { + handler, ok := handlers[operation] + if !ok { + return &Error{Code: "ValidationException", Message: "unknown operation " + operation} + } + body, err := json.Marshal(req) + if err != nil { + return err + } + r, err := http.NewRequestWithContext(ctx, http.MethodPost, "/"+operation, bytes.NewReader(body)) + if err != nil { + return err + } + if identity != "" { + r = r.WithContext(s3_constants.SetIdentityNameInContext(r.Context(), identity)) + } + w := httptest.NewRecorder() + handler(s, w, r) + if w.Code >= http.StatusBadRequest { + e := &Error{} + if json.Unmarshal(w.Body.Bytes(), e) != nil || e.Code == "" { + e = &Error{Code: http.StatusText(w.Code), Message: w.Body.String()} + } + return e + } + if resp == nil || w.Body.Len() == 0 { + return nil + } + return json.Unmarshal(w.Body.Bytes(), resp) +} diff --git a/weed/s3api/s3vectors/execute_test.go b/weed/s3api/s3vectors/execute_test.go new file mode 100644 index 0000000000..38f7508aa3 --- /dev/null +++ b/weed/s3api/s3vectors/execute_test.go @@ -0,0 +1,66 @@ +package s3vectors + +import ( + "context" + "errors" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" +) + +func TestExecuteRunsOperationsAsTheIdentity(t *testing.T) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), nil, nil) + s.SetRegion(s3tables.DefaultRegion) + s.SetAccountID(s3tables.DefaultAccountID) + ctx := context.Background() + admin := s3_constants.AccountAdminId + + var created CreateVectorBucketResponse + if err := s.Execute(ctx, "CreateVectorBucket", &CreateVectorBucketRequest{VectorBucketName: "media"}, &created, admin); err != nil { + t.Fatalf("create: %v", err) + } + if want := "arn:aws:s3vectors:us-east-1:000000000000:bucket/media"; created.VectorBucketArn != want { + t.Fatalf("arn %q, want %q", created.VectorBucketArn, want) + } + var list ListVectorBucketsResponse + if err := s.Execute(ctx, "ListVectorBuckets", &ListVectorBucketsRequest{}, &list, admin); err != nil { + t.Fatalf("list: %v", err) + } + if len(list.VectorBuckets) != 1 || list.VectorBuckets[0].VectorBucketName != "media" { + t.Fatalf("list = %+v", list.VectorBuckets) + } + + var e *Error + err := s.Execute(ctx, "GetVectorBucketPolicy", &GetVectorBucketPolicyRequest{VectorBucketName: "media"}, nil, admin) + if !errors.As(err, &e) || e.Code != "NotFoundException" { + t.Fatalf("policy of a bucket without one: %v, want NotFoundException", err) + } + if err := s.Execute(ctx, "DeleteVectorBucket", &DeleteVectorBucketRequest{VectorBucketName: "media"}, nil, admin); err != nil { + t.Fatalf("delete: %v", err) + } + err = s.Execute(ctx, "GetVectorBucket", &GetVectorBucketRequest{VectorBucketName: "media"}, nil, admin) + if !errors.As(err, &e) || e.Code != "NotFoundException" { + t.Fatalf("get after delete: %v, want NotFoundException", err) + } + err = s.Execute(ctx, "TagResource", struct{}{}, nil, admin) + if !errors.As(err, &e) || e.Code != "ValidationException" { + t.Fatalf("unknown operation: %v, want ValidationException", err) + } +} + +func TestExecuteChecksTheIdentitysGrants(t *testing.T) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), nil, nil) + ctx := context.Background() + if err := s.Execute(ctx, "CreateVectorBucket", &CreateVectorBucketRequest{VectorBucketName: "media"}, nil, s3_constants.AccountAdminId); err != nil { + t.Fatalf("create: %v", err) + } + var e *Error + err := s.Execute(ctx, "DeleteVectorBucket", &DeleteVectorBucketRequest{VectorBucketName: "media"}, nil, "someone-else") + if !errors.As(err, &e) || e.Code != "AccessDeniedException" { + t.Fatalf("delete by a stranger: %v, want AccessDeniedException", err) + } +} diff --git a/weed/s3api/s3vectors/handlers.go b/weed/s3api/s3vectors/handlers.go index 9b3704aa60..0f629d2cc1 100644 --- a/weed/s3api/s3vectors/handlers.go +++ b/weed/s3api/s3vectors/handlers.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "net/http" "strconv" "strings" @@ -38,11 +39,18 @@ var handlers = map[string]func(*Server, http.ResponseWriter, *http.Request){ "ListVectors": (*Server).handleListVectors, "DeleteVectors": (*Server).handleDeleteVectors, "QueryVectors": (*Server).handleQueryVectors, + "TagResource": (*Server).handleTagResource, + "UntagResource": (*Server).handleUntagResource, + "ListTagsForResource": (*Server).handleListTagsForResource, } // lance table ids put indexes under a "vector" namespace inside their bucket. const indexNamespace = s3tables.VectorIndexNamespace +// ensureIndexNamespaceAttempts bounds how often ensureIndexNamespace declares +// the namespace again after losing it to a concurrent declare. +const ensureIndexNamespaceAttempts = 3 + // Vector bucket records are s3tables table buckets; the S3 Vectors ARN form is // arn:aws:s3vectors:{region}:{account}:bucket/{name}[/index/{name}]. func (s *Server) vectorBucketArn(bucket string) string { @@ -178,7 +186,8 @@ func catalogErrCode(err error) string { func catalogErr(w http.ResponseWriter, err error) { var apiErr *s3tables.S3TablesError if !errors.As(err, &apiErr) { - writeError(w, http.StatusInternalServerError, "InternalFailure", "internal error") + glog.Errorf("S3 Vectors: catalog: %v", err) + writeError(w, http.StatusInternalServerError, "InternalFailure", internalErrorMessage) return } switch { @@ -196,10 +205,12 @@ func catalogErr(w http.ResponseWriter, err error) { } } +const internalErrorMessage = "internal error, see the S3 gateway log" + // vectorErr maps a VectorService gRPC status onto the API's error shape. // Engine messages reach clients only for codes that describe the request // itself; transport and internal details stay in the server log. -func vectorErr(w http.ResponseWriter, err error) { +func vectorErr(w http.ResponseWriter, r *http.Request, arns vectorBucketARNs, err error) { msg := status.Convert(err).Message() switch status.Code(err) { case codes.NotFound: @@ -209,10 +220,11 @@ func vectorErr(w http.ResponseWriter, err error) { case codes.PermissionDenied, codes.Unauthenticated: writeError(w, http.StatusForbidden, "AccessDeniedException", "access denied") case codes.Unavailable: + glog.Warningf("S3 Vectors: %s %s/%s: %v", operationFromRequest(r), arns.bucket, arns.index, err) writeError(w, http.StatusServiceUnavailable, "ServiceUnavailableException", "vector service unavailable") default: - glog.V(1).Infof("S3 Vectors: engine error: %v", err) - writeError(w, http.StatusInternalServerError, "InternalFailure", "internal error") + glog.Errorf("S3 Vectors: %s %s/%s: %v", operationFromRequest(r), arns.bucket, arns.index, err) + writeError(w, http.StatusInternalServerError, "InternalFailure", internalErrorMessage) } } @@ -220,26 +232,36 @@ func vectorErr(w http.ResponseWriter, err error) { // calls are internal: a caller's CreateTable grant is what authorizes the // index, so it must not also need namespace reads or writes. func (s *Server) ensureIndexNamespace(ctx context.Context, bucket string) error { - var resp map[string]interface{} - err := s.internalCatalog(ctx, "GetNamespace", &s3tables.GetNamespaceRequest{ - TableBucketARN: s.s3tablesArn(bucket), - Namespace: []string{indexNamespace}, - }, &resp) - if err == nil { - return nil + var err error + for attempt := 0; attempt < ensureIndexNamespaceAttempts; attempt++ { + if attempt > 0 { + time.Sleep(time.Duration(attempt) * 50 * time.Millisecond) + } + var resp map[string]interface{} + err = s.internalCatalog(ctx, "GetNamespace", &s3tables.GetNamespaceRequest{ + TableBucketARN: s.s3tablesArn(bucket), + Namespace: []string{indexNamespace}, + }, &resp) + if err == nil { + return nil + } + if code := catalogErrCode(err); code != s3tables.ErrCodeNoSuchNamespace && code != s3tables.ErrCodeNoSuchBucket { + return err + } + err = s.internalCatalog(ctx, "CreateNamespace", &s3tables.CreateNamespaceRequest{ + TableBucketARN: s.s3tablesArn(bucket), + Namespace: []string{indexNamespace}, + }, &resp) + // A concurrent CreateIndex may have won the create, or taken over this + // one's namespace before it was confirmed. + if err == nil || catalogErrCode(err) == s3tables.ErrCodeNamespaceAlreadyExists { + return nil + } + if catalogErrCode(err) != s3tables.ErrCodeConflict { + return err + } } - if code := catalogErrCode(err); code != s3tables.ErrCodeNoSuchNamespace && code != s3tables.ErrCodeNoSuchBucket { - return err - } - err = s.internalCatalog(ctx, "CreateNamespace", &s3tables.CreateNamespaceRequest{ - TableBucketARN: s.s3tablesArn(bucket), - Namespace: []string{indexNamespace}, - }, &resp) - // A concurrent CreateIndex may have won the create. - if err != nil && catalogErrCode(err) != s3tables.ErrCodeNamespaceAlreadyExists { - return err - } - return nil + return err } func (s *Server) handleCreateVectorBucket(w http.ResponseWriter, r *http.Request) { @@ -256,6 +278,10 @@ func (s *Server) handleCreateVectorBucket(w http.ResponseWriter, r *http.Request writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } + if err := s3tables.ValidateTagsWithLimit(req.Tags, maxResourceTags); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } // The catalog keeps the s3tables field name; GetVectorBucket renames it back. enc := req.EncryptionConfiguration if len(enc) != 0 { @@ -263,18 +289,23 @@ func (s *Server) handleCreateVectorBucket(w http.ResponseWriter, r *http.Request } // No bucket exists yet, so check against the handler account as the // catalog does for CreateTableBucket. - principal := s.tables.PrincipalFor(identityOf(r)) - if !s3tables.CheckPermissionWithContext("s3vectors:CreateVectorBucket", principal, s.tables.PrincipalFor(""), "", "", s.tables.PolicyContext(r)) { + principal := s.tables.PrincipalFor(r) + allowed := func(action, bucket string) bool { + return s3tables.CheckPermissionWithContext(action, principal, s.tables.PrincipalFor(nil), "", "", s.tables.PolicyContext(r, bucket)) + } + // Tagging names the new bucket, so a grant scoped to it applies. + if !allowed("s3vectors:CreateVectorBucket", "") || + (len(req.Tags) > 0 && !allowed("s3vectors:TagResource", req.VectorBucketName)) { catalogErr(w, errVectorDenied) return } - // The caller's name still makes it the owner of the record. var resp map[string]interface{} err := s.internal.Execute(r.Context(), s.filerClient, "CreateTableBucket", &s3tables.CreateTableBucketRequest{ Name: req.VectorBucketName, Format: s3tables.FormatLance, EncryptionConfiguration: enc, + Tags: req.Tags, }, &resp, identityOf(r)) if err != nil { catalogErr(w, err) @@ -346,13 +377,12 @@ func (s *Server) handleListVectorBuckets(w http.ResponseWriter, r *http.Request) catalogErr(w, err) return } - principal := s.tables.PrincipalFor(identityOf(r)) - pctx := s.tables.PolicyContext(r) + principal := s.tables.PrincipalFor(r) // The catalog token continues the filtered page: ICEBERG buckets drop out // of the listing, Lance and format-less legacy buckets stay. out := &ListVectorBucketsResponse{NextToken: resp.ContinuationToken, VectorBuckets: []*VectorBucket{}} for _, b := range resp.TableBuckets { - if b.Format != "" && b.Format != s3tables.FormatLance { + if !IsVectorBucketFormat(b.Format) { continue } owner, policy, err := s.vectorPolicyView(r.Context(), b.Name) @@ -362,6 +392,7 @@ func (s *Server) handleListVectorBuckets(w http.ResponseWriter, r *http.Request) continue } arn := s.vectorBucketArn(b.Name) + pctx := s.tables.PolicyContext(r, b.Name) if !s3tables.CheckPermissionWithContext("s3vectors:ListVectorBuckets", principal, owner, policy, arn, pctx) && !s3tables.CheckPermissionWithContext("s3vectors:GetVectorBucket", principal, owner, policy, arn, pctx) { continue @@ -387,25 +418,29 @@ func (s *Server) handleDeleteVectorBucket(w http.ResponseWriter, r *http.Request writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - if _, err := s.authorizeVector(r.Context(), r, "s3vectors:DeleteVectorBucket", arns.bucket, ""); err != nil { + record, err := s.authorizeVector(r.Context(), r, "s3vectors:DeleteVectorBucket", arns.bucket, "") + if err != nil { catalogErr(w, err) return } + // Only the bucket authorized above is deleted, not one created again + // under its name since. + ctx := s3tables.WithTableBucketCreatedAt(r.Context(), record.CreatedAt) var resp map[string]interface{} - err = s.internalCatalog(r.Context(), "DeleteTableBucket", + err = s.internalCatalog(ctx, "DeleteTableBucket", &s3tables.DeleteTableBucketRequest{TableBucketARN: s.s3tablesArn(arns.bucket)}, &resp) if err != nil && catalogErrCode(err) == s3tables.ErrCodeBucketNotEmpty { // The implicit index namespace counts as bucket content even when it // holds no indexes; drop it under the delete's own authorization and // retry once. A namespace still holding indexes keeps the conflict. var nsResp map[string]interface{} - nsErr := s.internalCatalog(context.WithoutCancel(r.Context()), "DeleteNamespace", + nsErr := s.internalCatalog(context.WithoutCancel(ctx), "DeleteNamespace", &s3tables.DeleteNamespaceRequest{ TableBucketARN: s.s3tablesArn(arns.bucket), Namespace: []string{indexNamespace}, }, &nsResp) if nsErr == nil { - err = s.internalCatalog(r.Context(), "DeleteTableBucket", + err = s.internalCatalog(ctx, "DeleteTableBucket", &s3tables.DeleteTableBucketRequest{TableBucketARN: s.s3tablesArn(arns.bucket)}, &resp) } else if code := catalogErrCode(nsErr); code != s3tables.ErrCodeNoSuchNamespace { glog.V(1).Infof("S3 Vectors: drop internal namespace in %s: %v", arns.bucket, nsErr) @@ -723,25 +758,41 @@ func (s *Server) mutateVectorPolicy(ctx context.Context, bucket string, mutate f }) } -// isVectorBucket reports whether the catalog record describes a vector -// bucket: an explicit ICEBERG bucket is not one, while a bucket declared -// before formats existed reads as Lance. +// IsVectorBucketFormat reports whether a table bucket of this format is a +// vector bucket: an explicit ICEBERG bucket is not one, while a bucket +// declared before formats existed reads as Lance. +func IsVectorBucketFormat(format string) bool { + return format == "" || format == s3tables.FormatLance +} + func isVectorBucket(record *s3tables.GetTableBucketResponse) bool { - return record.Format == "" || record.Format == s3tables.FormatLance + return IsVectorBucketFormat(record.Format) } // errVectorDenied marks a failed vector permission check inside a policy // mutation; catalogErr renders it as AccessDenied. var errVectorDenied = &s3tables.S3TablesError{Type: s3tables.ErrCodeAccessDenied, Message: "access denied"} +var errVectorBucketReplaced = &s3tables.S3TablesError{Type: s3tables.ErrCodeConflict, + Message: "vector bucket changed concurrently, retry the request"} + // checkVectorPolicyDoc evaluates the caller's vector action against the // document the entry currently holds. Called inside a policy mutation's // compare-and-swap, so a grant revoked between reads never authorizes the -// pending write. -func (s *Server) checkVectorPolicyDoc(r *http.Request, action, bucket, owner string, ext map[string][]byte) bool { - principal := s.tables.PrincipalFor(identityOf(r)) - return s3tables.CheckPermissionWithContext(action, principal, owner, - vectorPolicyFromExt(ext), s.vectorBucketArn(bucket), s.tables.PolicyContext(r)) +// pending write, and the entry must still be the bucket record was read from. +func (s *Server) checkVectorPolicyDoc(r *http.Request, action, bucket string, record *s3tables.GetTableBucketResponse, ext map[string][]byte) error { + var meta struct { + CreatedAt time.Time `json:"createdAt"` + } + if json.Unmarshal(ext[s3tables.ExtendedKeyMetadata], &meta) != nil || !meta.CreatedAt.Equal(record.CreatedAt) { + return errVectorBucketReplaced + } + principal := s.tables.PrincipalFor(r) + if !s3tables.CheckPermissionWithContext(action, principal, record.OwnerAccountID, + vectorPolicyFromExt(ext), s.vectorBucketArn(bucket), s.tables.PolicyContext(r, bucket)) { + return errVectorDenied + } + return nil } // authorizeVector grants the request's own vector operation — not a catalog @@ -763,8 +814,8 @@ func (s *Server) authorizeVector(ctx context.Context, r *http.Request, action, b if index != "" { resourceARN = s.indexArn(bucket, index) } - principal := s.tables.PrincipalFor(identityOf(r)) - if !s3tables.CheckPermissionWithContext(action, principal, record.OwnerAccountID, policy, resourceARN, s.tables.PolicyContext(r)) { + principal := s.tables.PrincipalFor(r) + if !s3tables.CheckPermissionWithContext(action, principal, record.OwnerAccountID, policy, resourceARN, s.tables.PolicyContext(r, bucket)) { return nil, &s3tables.S3TablesError{Type: s3tables.ErrCodeAccessDenied, Message: "access denied"} } if !isVectorBucket(record) { @@ -800,8 +851,8 @@ func (s *Server) handlePutVectorBucketPolicy(w http.ResponseWriter, r *http.Requ return } err = s.mutateVectorPolicy(r.Context(), arns.bucket, func(ext map[string][]byte) error { - if !s.checkVectorPolicyDoc(r, "s3vectors:PutVectorBucketPolicy", arns.bucket, record.OwnerAccountID, ext) { - return errVectorDenied + if err := s.checkVectorPolicyDoc(r, "s3vectors:PutVectorBucketPolicy", arns.bucket, record, ext); err != nil { + return err } // A raw vector document still in the shared slot migrates out with // this write; catalog rules in it stay untouched. @@ -871,8 +922,8 @@ func (s *Server) handleDeleteVectorBucketPolicy(w http.ResponseWriter, r *http.R return } err = s.mutateVectorPolicy(r.Context(), arns.bucket, func(ext map[string][]byte) error { - if !s.checkVectorPolicyDoc(r, "s3vectors:DeleteVectorBucketPolicy", arns.bucket, record.OwnerAccountID, ext) { - return errVectorDenied + if err := s.checkVectorPolicyDoc(r, "s3vectors:DeleteVectorBucketPolicy", arns.bucket, record, ext); err != nil { + return err } removed := false if _, ok := ext[vectorPolicyKey]; ok { @@ -929,14 +980,27 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", "dimension must be 1..=4096") return } + if err := s3tables.ValidateTagsWithLimit(req.Tags, maxResourceTags); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } // The create grant is exercised before any write: a denied caller must not // leave even the implicit namespace behind. - if _, err := s.authorizeVector(r.Context(), r, "s3vectors:CreateIndex", arns.bucket, arns.index); err != nil { + bucketRecord, err := s.authorizeVector(r.Context(), r, "s3vectors:CreateIndex", arns.bucket, arns.index) + if err != nil { catalogErr(w, err) return } + if len(req.Tags) > 0 { + if _, err := s.authorizeVector(r.Context(), r, "s3vectors:TagResource", arns.bucket, arns.index); err != nil { + catalogErr(w, err) + return + } + } + // The catalog writes land only in the bucket authorized above. + bound := s3tables.WithTableBucketCreatedAt(r.Context(), bucketRecord.CreatedAt) // The namespace is implicit on AWS and internal bookkeeping here. - if err := s.ensureIndexNamespace(r.Context(), arns.bucket); err != nil { + if err := s.ensureIndexNamespace(bound, arns.bucket); err != nil { catalogErr(w, err) return } @@ -971,12 +1035,13 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) { } } metadataJSON, _ := json.Marshal(metadata) - err = s.internalCatalog(r.Context(), "CreateTable", &s3tables.CreateTableRequest{ + err = s.internalCatalog(bound, "CreateTable", &s3tables.CreateTableRequest{ TableBucketARN: s.s3tablesArn(arns.bucket), Namespace: []string{indexNamespace}, Name: arns.index, Format: s3tables.FormatLance, Metadata: &s3tables.TableMetadata{FullMetadata: metadataJSON}, + Tags: req.Tags, }, nil) if err != nil { catalogErr(w, err) @@ -1012,7 +1077,7 @@ func (s *Server) handleCreateIndex(w http.ResponseWriter, r *http.Request) { } else if gerr != nil { glog.Errorf("S3 Vectors: rollback lookup for %s/%s failed: %v", arns.bucket, arns.index, gerr) } - vectorErr(w, err) + vectorErr(w, r, arns, err) return } writeJSON(w, http.StatusOK, &CreateIndexResponse{IndexArn: s.indexArn(arns.bucket, arns.index)}) @@ -1171,7 +1236,8 @@ func (s *Server) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) return } - if _, err := s.authorizeVector(r.Context(), r, "s3vectors:DeleteIndex", arns.bucket, arns.index); err != nil { + bucketRecord, err := s.authorizeVector(r.Context(), r, "s3vectors:DeleteIndex", arns.bucket, arns.index) + if err != nil { catalogErr(w, err) return } @@ -1195,8 +1261,9 @@ func (s *Server) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { // Deleting the catalog record is the delete: the catalog purges the // dataset under the index's location along with the entry. var resp map[string]interface{} - // The token limits the delete to the record checked above. - err = s.internalCatalog(r.Context(), "DeleteTable", &s3tables.DeleteTableRequest{ + // The token limits the delete to the record checked above, and the bound + // bucket to the one authorized. + err = s.internalCatalog(s3tables.WithTableBucketCreatedAt(r.Context(), bucketRecord.CreatedAt), "DeleteTable", &s3tables.DeleteTableRequest{ TableBucketARN: s.s3tablesArn(arns.bucket), Namespace: []string{indexNamespace}, Name: arns.index, @@ -1265,7 +1332,7 @@ func (s *Server) handlePutVectors(w http.ResponseWriter, r *http.Request) { Vectors: vectors, }) if err != nil { - vectorErr(w, err) + vectorErr(w, r, arns, err) return } writeJSON(w, http.StatusOK, &PutVectorsResponse{}) @@ -1346,7 +1413,7 @@ func (s *Server) handleGetVectors(w http.ResponseWriter, r *http.Request) { ReturnMetadata: req.ReturnMetadata, }) if err != nil { - vectorErr(w, err) + vectorErr(w, r, arns, err) return } writeJSON(w, http.StatusOK, &GetVectorsResponse{ @@ -1405,7 +1472,7 @@ func (s *Server) handleListVectors(w http.ResponseWriter, r *http.Request) { } resp, err := s.vectors.ListVectors(r.Context(), listReq) if err != nil { - vectorErr(w, err) + vectorErr(w, r, arns, err) return } // An engine that predates segment_count lists every row; page through @@ -1415,7 +1482,7 @@ func (s *Server) handleListVectors(w http.ResponseWriter, r *http.Request) { out, nextToken, err := s.listSegmentVectors(r.Context(), listReq.Table, offset, maxResults, uint64(*req.SegmentCount), uint64(*req.SegmentIndex), req.ReturnData, req.ReturnMetadata, resp) if err != nil { - vectorErr(w, err) + vectorErr(w, r, arns, err) return } writeJSON(w, http.StatusOK, &ListVectorsResponse{ @@ -1545,7 +1612,7 @@ func (s *Server) handleDeleteVectors(w http.ResponseWriter, r *http.Request) { Keys: req.Keys, }) if err != nil { - vectorErr(w, err) + vectorErr(w, r, arns, err) return } writeJSON(w, http.StatusOK, &DeleteVectorsResponse{}) @@ -1698,16 +1765,12 @@ func fieldClause(field string, cond interface{}) (string, error) { return "", fmt.Errorf("$exists expects a boolean") } case "$eq", "$ne": - lit, err := sqlLiteral(operand) - if err != nil { - return "", err - } - match, err := fieldMatch(field, operand, "= "+lit) + match, err := equalMatch(field, operand) if err != nil { return "", err } if op == "$ne" { - match = "NOT COALESCE(" + match + ", false)" + match = "NOT " + match } parts = append(parts, match) case "$in", "$nin": @@ -1747,6 +1810,13 @@ func fieldClause(field string, cond interface{}) (string, error) { for _, col := range cols { groups = append(groups, fmt.Sprintf("COALESCE(%s in (%s), false)", col, strings.Join(members[col], ", "))) } + for _, item := range list { + element, err := elementMatch(field, item) + if err != nil { + return "", err + } + groups = append(groups, element) + } expr := "(" + strings.Join(groups, " OR ") + ")" if op == "$nin" { expr = "NOT " + expr @@ -1775,14 +1845,41 @@ func fieldClause(field string, cond interface{}) (string, error) { } return strings.Join(parts, " AND "), nil default: - lit, err := sqlLiteral(cond) - if err != nil { - return "", err - } - return fieldMatch(field, cond, "= "+lit) + return equalMatch(field, cond) } } +// equalMatch renders "field equals operand": the stored value itself or, as +// AWS documents for $eq, any element of a stored array. +func equalMatch(field string, operand interface{}) (string, error) { + lit, err := sqlLiteral(operand) + if err != nil { + return "", err + } + match, err := fieldMatch(field, operand, "= "+lit) + if err != nil { + return "", err + } + element, err := elementMatch(field, operand) + if err != nil { + return "", err + } + return "(COALESCE(" + match + ", false) OR " + element + ")", nil +} + +// elementMatch renders "the array stored at field holds operand". The engine +// compares the elements as typed JSON values, so 1 does not match "1". +func elementMatch(field string, operand interface{}) (string, error) { + var b strings.Builder + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) + if err := enc.Encode(operand); err != nil { + return "", err + } + return fmt.Sprintf("COALESCE(json_array_contains(`metadata`, '%s', '%s'), false)", + sqlEscape(existsPath(field)), sqlEscape(strings.TrimSuffix(b.String(), "\n"))), nil +} + // jsonbString renders s as the JSON text json_extract returns for a stored // string: quoted, with the same escapes the jsonb serializer emits. func jsonbString(s string) string { @@ -1888,14 +1985,19 @@ func (s *Server) handleQueryVectors(w http.ResponseWriter, r *http.Request) { ReturnDistance: req.ReturnDistance, }) if err != nil { - vectorErr(w, err) + vectorErr(w, r, arns, err) return } out := &QueryVectorsResponse{DistanceMetric: metric, Vectors: []*QueryOutputVector{}} for _, v := range resp.Vectors { o := &QueryOutputVector{Key: v.Key} if req.ReturnDistance { - o.Distance = &v.Distance + d := v.Distance + if metric == "euclidean" { + // Lance's L2 distance is the square of the Euclidean distance. + d = float32(math.Sqrt(math.Max(0, float64(d)))) + } + o.Distance = &d } if req.ReturnMetadata { o.Metadata = structToMap(v.Metadata) diff --git a/weed/s3api/s3vectors/handlers_test.go b/weed/s3api/s3vectors/handlers_test.go index 7aad7447bb..6ed043d8bc 100644 --- a/weed/s3api/s3vectors/handlers_test.go +++ b/weed/s3api/s3vectors/handlers_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "strconv" "strings" + "sync" "testing" "time" @@ -18,6 +19,8 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // fakeVectors serves a fixed row set with the engine's offset-token paging @@ -352,19 +355,26 @@ func TestResolveIndexARNs(t *testing.T) { } func TestTranslateFilter(t *testing.T) { + str := "json_extract(`metadata`, '$[\"k\"]')" + num := "TRY_CAST(json_extract(`metadata`, '$[\"k\"]') AS DOUBLE)" + elem := func(value string) string { + return "COALESCE(json_array_contains(`metadata`, '$[\"k\"]', '" + value + "'), false)" + } cases := []struct { name string filter map[string]interface{} want string }{ {"bare string equality", map[string]interface{}{"k": "v"}, - "json_extract(`metadata`, '$[\"k\"]') = '\"v\"'"}, + "(COALESCE(" + str + " = '\"v\"', false) OR " + elem(`"v"`) + ")"}, {"bare number equality", map[string]interface{}{"k": float64(3)}, - "TRY_CAST(json_extract(`metadata`, '$[\"k\"]') AS DOUBLE) = 3"}, + "(COALESCE(" + num + " = 3, false) OR " + elem("3") + ")"}, {"bare bool equality", map[string]interface{}{"k": true}, - "json_extract(`metadata`, '$[\"k\"]') = 'true'"}, + "(COALESCE(" + str + " = 'true', false) OR " + elem("true") + ")"}, + {"eq fraction", map[string]interface{}{"k": map[string]interface{}{"$eq": float64(2.5)}}, + "(COALESCE(" + num + " = 2.5, false) OR " + elem("2.5") + ")"}, {"ne", map[string]interface{}{"k": map[string]interface{}{"$ne": "v"}}, - "NOT COALESCE(json_extract(`metadata`, '$[\"k\"]') = '\"v\"', false)"}, + "NOT (COALESCE(" + str + " = '\"v\"', false) OR " + elem(`"v"`) + ")"}, {"exists true", map[string]interface{}{"k": map[string]interface{}{"$exists": true}}, "json_exists(`metadata`, '$[\"k\"]')"}, {"exists false", map[string]interface{}{"k": map[string]interface{}{"$exists": false}}, @@ -372,19 +382,23 @@ func TestTranslateFilter(t *testing.T) { {"exists special key", map[string]interface{}{`a.b"c`: map[string]interface{}{"$exists": true}}, "json_exists(`metadata`, '$[\"a.b\\\"c\"]')"}, {"range", map[string]interface{}{"k": map[string]interface{}{"$gt": float64(1)}}, - "TRY_CAST(json_extract(`metadata`, '$[\"k\"]') AS DOUBLE) > 1"}, + num + " > 1"}, {"in strings", map[string]interface{}{"k": map[string]interface{}{"$in": []interface{}{"a", "b"}}}, - "(COALESCE(json_extract(`metadata`, '$[\"k\"]') in ('\"a\"', '\"b\"'), false))"}, + "(COALESCE(" + str + " in ('\"a\"', '\"b\"'), false) OR " + elem(`"a"`) + " OR " + elem(`"b"`) + ")"}, {"nin numbers", map[string]interface{}{"k": map[string]interface{}{"$nin": []interface{}{float64(1)}}}, - "NOT (COALESCE(TRY_CAST(json_extract(`metadata`, '$[\"k\"]') AS DOUBLE) in (1), false))"}, + "NOT (COALESCE(" + num + " in (1), false) OR " + elem("1") + ")"}, {"in mixed types", map[string]interface{}{"k": map[string]interface{}{"$in": []interface{}{"a", float64(1), true, float64(2)}}}, - "(COALESCE(json_extract(`metadata`, '$[\"k\"]') in ('\"a\"', 'true'), false) OR COALESCE(TRY_CAST(json_extract(`metadata`, '$[\"k\"]') AS DOUBLE) in (1, 2), false))"}, + "(COALESCE(" + str + " in ('\"a\"', 'true'), false) OR COALESCE(" + num + " in (1, 2), false) OR " + + elem(`"a"`) + " OR " + elem("1") + " OR " + elem("true") + " OR " + elem("2") + ")"}, {"and", map[string]interface{}{"$and": []interface{}{ map[string]interface{}{"a": "1"}, map[string]interface{}{"b": float64(2)}, }}, - "(json_extract(`metadata`, '$[\"a\"]') = '\"1\"' AND TRY_CAST(json_extract(`metadata`, '$[\"b\"]') AS DOUBLE) = 2)"}, + "((COALESCE(json_extract(`metadata`, '$[\"a\"]') = '\"1\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"a\"]', '\"1\"'), false)) AND " + + "(COALESCE(TRY_CAST(json_extract(`metadata`, '$[\"b\"]') AS DOUBLE) = 2, false) OR COALESCE(json_array_contains(`metadata`, '$[\"b\"]', '2'), false)))"}, {"sql injection", map[string]interface{}{"a'b": "v'1"}, - "json_extract(`metadata`, '$[\"a''b\"]') = '\"v''1\"'"}, + "(COALESCE(json_extract(`metadata`, '$[\"a''b\"]') = '\"v''1\"', false) OR COALESCE(json_array_contains(`metadata`, '$[\"a''b\"]', '\"v''1\"'), false))"}, + {"array value escapes", map[string]interface{}{"k": ``}, + "(COALESCE(" + str + ` = '""', false) OR ` + elem(`""`) + ")"}, } for _, tc := range cases { got, err := translateFilter(tc.filter) @@ -692,6 +706,32 @@ func TestValidateEncryptionConfigurationRejectsCustomerKeys(t *testing.T) { } } +// The gateway applies a vector bucket's AES256 to every index write, so a +// gateway without an SSE-S3 key refuses the bucket instead of failing each +// later write. +func TestCreateVectorBucketNeedsSSES3ForAES256(t *testing.T) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, newFakeVectors(0)) + keyLoaded := false + s.SetEncryptionChecker(func() bool { return keyLoaded }) + aes := map[string]interface{}{"sseType": "AES256"} + + w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "sealed", EncryptionConfiguration: aes}) + if w.Code != http.StatusBadRequest || !strings.Contains(w.Body.String(), "ValidationException") || !strings.Contains(w.Body.String(), "s3.sse.key") { + t.Fatalf("AES256 without a key: %d %s, want 400 ValidationException naming s3.sse.key", w.Code, w.Body.String()) + } + if w := postJSON(t, s.handleGetVectorBucket, &GetVectorBucketRequest{VectorBucketName: "sealed"}); w.Code != http.StatusNotFound { + t.Fatalf("refused bucket exists: %d %s", w.Code, w.Body.String()) + } + if w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "plain"}); w.Code != http.StatusOK { + t.Fatalf("unencrypted bucket without a key: %d %s", w.Code, w.Body.String()) + } + keyLoaded = true + if w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "sealed", EncryptionConfiguration: aes}); w.Code != http.StatusOK { + t.Fatalf("AES256 with a key: %d %s", w.Code, w.Body.String()) + } +} + // A vector index is colocated: its catalog entry sits at the dataset root, so // the catalog delete removes entry and data together. func TestDeleteIndexRemovesDatasetDirectory(t *testing.T) { @@ -983,3 +1023,83 @@ func TestVectorKeysBoundsRejectsBeforeEngine(t *testing.T) { t.Fatalf("engine saw %d get/%d delete calls, want none", len(f.gets), len(f.deletes)) } } + +// Two indexes created at once in a new bucket both declare the implicit +// namespace. One create takes over the other's namespace while it is still +// marked unfinished, so the other's confirmation fails; its index is created +// all the same. +func TestCreateIndexSurvivesAConcurrentNamespaceCreate(t *testing.T) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), allowAllAuth{}, newFakeVectors(0)) + if w := postJSON(t, s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}); w.Code != http.StatusOK { + t.Fatalf("create bucket: %d %s", w.Code, w.Body.String()) + } + bucketPath := s3tables.GetTableBucketPath("vec-bucket") + reached := make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + releaseFirst := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(releaseFirst) + held := false + // Hold the first create where it confirms the bucket after writing the namespace. + fs.OnLookup = func(dir, name string) { + if held || dir != s3tables.TablesPath || name != "vec-bucket" { + return + } + ns := fs.Get(bucketPath, indexNamespace) + if ns == nil || string(ns.Extended[s3tables.ExtendedKeyDeleting]) != "create" { + return + } + held = true + close(reached) + <-release + } + createIndex := func(name string) *httptest.ResponseRecorder { + return postJSON(t, s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: "vec-bucket", IndexName: name, DataType: "float32", Dimension: 2, DistanceMetric: "cosine", + }) + } + first := make(chan *httptest.ResponseRecorder, 1) + go func() { first <- createIndex("first") }() + select { + case <-reached: + case <-time.After(10 * time.Second): + t.Fatal("the first create never wrote an unfinished namespace") + } + + if w := createIndex("second"); w.Code != http.StatusOK { + t.Fatalf("second create index: %d %s", w.Code, w.Body.String()) + } + releaseFirst() + if w := <-first; w.Code != http.StatusOK { + t.Fatalf("first create index: %d %s", w.Code, w.Body.String()) + } +} + +func TestVectorErrKeepsEngineDetailsOutOfTheResponse(t *testing.T) { + arns := vectorBucketARNs{bucket: "media", index: "movies"} + for _, tc := range []struct { + err error + wantStatus int + wantCode string + wantMessage string + }{ + {status.Error(codes.Internal, "declare: 500 from http://10.0.0.5:8888/lance/v1/table/media$vector$movies/declare"), + http.StatusInternalServerError, "InternalFailure", internalErrorMessage}, + {status.Error(codes.Unavailable, "no weed-vector registered on master"), + http.StatusServiceUnavailable, "ServiceUnavailableException", "vector service unavailable"}, + {status.Error(codes.InvalidArgument, "vector has dimension 2, index has 3"), + http.StatusBadRequest, "ValidationException", "vector has dimension 2, index has 3"}, + } { + rec := httptest.NewRecorder() + vectorErr(rec, httptest.NewRequest(http.MethodPost, "/PutVectors", nil), arns, tc.err) + var body map[string]string + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if rec.Code != tc.wantStatus || body["code"] != tc.wantCode || body["message"] != tc.wantMessage { + t.Errorf("%v: got %d %s %q, want %d %s %q", status.Code(tc.err), rec.Code, body["code"], body["message"], + tc.wantStatus, tc.wantCode, tc.wantMessage) + } + } +} diff --git a/weed/s3api/s3vectors/principal_test.go b/weed/s3api/s3vectors/principal_test.go new file mode 100644 index 0000000000..7b3ee431a5 --- /dev/null +++ b/weed/s3api/s3vectors/principal_test.go @@ -0,0 +1,240 @@ +package s3vectors + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/s3api" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" +) + +// authTestEnv drives the vector API through Auth with a chosen identity and +// the table bucket API through the catalog with the same identity, so the +// two can be compared decision by decision. +type authTestEnv struct { + t *testing.T + s *Server + auth *fakeAuthenticator + tables *s3tables.Manager + admin *s3tables.Manager + client s3tables.FilerClient +} + +func newAuthTestEnv(t *testing.T) *authTestEnv { + fs := s3tablestest.Start(t) + client := s3tables.NewManagerClient(fs.Client) + auth := &fakeAuthenticator{} + admin := s3tables.NewManager() + admin.SetTrusted(true) + return &authTestEnv{t: t, s: NewServer(client, auth, newFakeVectors(0)), auth: auth, + tables: s3tables.NewManager(), admin: admin, client: client} +} + +func (e *authTestEnv) vector(ident *s3api.Identity, handler http.HandlerFunc, payload interface{}) int { + e.t.Helper() + return e.vectorResp(ident, handler, payload).Code +} + +func (e *authTestEnv) vectorResp(ident *s3api.Identity, handler http.HandlerFunc, payload interface{}) *httptest.ResponseRecorder { + e.t.Helper() + body, err := json.Marshal(payload) + if err != nil { + e.t.Fatalf("marshal request: %v", err) + } + e.auth.identity = ident + w := httptest.NewRecorder() + e.s.Auth(handler)(w, httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body))) + return w +} + +func (e *authTestEnv) table(ident *s3api.Identity, operation string, req interface{}) error { + ctx := s3_constants.SetIdentityInContext(contextWithIdentity(context.Background(), ident.Name), ident) + var resp map[string]interface{} + return e.tables.Execute(ctx, e.client, operation, req, &resp, ident.Name) +} + +func tableARN(bucket string) string { + return "arn:aws:s3tables:us-east-1:000000000000:bucket/" + bucket +} + +func denyPolicy(service, principal string) string { + return fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Principal":{"AWS":%q},"Action":"%s:*","Resource":"*"}]}`, + principal, service) +} + +func (e *authTestEnv) createBuckets(ident *s3api.Identity, name string) { + e.t.Helper() + if err := e.table(ident, "CreateTableBucket", &s3tables.CreateTableBucketRequest{Name: "t-" + name}); err != nil { + e.t.Fatalf("%s: create table bucket: %v", ident.Name, err) + } + if code := e.vector(ident, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "v-" + name}); code != http.StatusOK { + e.t.Fatalf("%s: create vector bucket: %d", ident.Name, code) + } +} + +func (e *authTestEnv) deny(name, principal string) { + e.t.Helper() + var resp map[string]interface{} + if err := e.admin.Execute(context.Background(), e.client, "PutTableBucketPolicy", &s3tables.PutTableBucketPolicyRequest{ + TableBucketARN: tableARN("t-" + name), ResourcePolicy: denyPolicy("s3tables", principal), + }, &resp, ""); err != nil { + e.t.Fatalf("put table policy: %v", err) + } + if code := e.vector(adminIdentity(), e.s.handlePutVectorBucketPolicy, &PutVectorBucketPolicyRequest{ + VectorBucketName: "v-" + name, Policy: denyPolicy("s3vectors", principal), + }); code != http.StatusOK { + e.t.Fatalf("put vector policy: %d", code) + } +} + +// canGet reports whether ident may read the table bucket and the vector +// bucket created under name. +func (e *authTestEnv) canGet(ident *s3api.Identity, name string) (table, vector bool) { + e.t.Helper() + table = e.table(ident, "GetTableBucket", &s3tables.GetTableBucketRequest{TableBucketARN: tableARN("t-" + name)}) == nil + vector = e.vector(ident, e.s.handleGetVectorBucket, &GetVectorBucketRequest{VectorBucketName: "v-" + name}) == http.StatusOK + return table, vector +} + +func adminIdentity() *s3api.Identity { + return &s3api.Identity{Name: "admin", Account: &s3api.AccountAdmin, Actions: []s3api.Action{s3_constants.ACTION_ADMIN}} +} + +var createActions = []s3api.Action{"s3tables:CreateTableBucket", "s3vectors:CreateVectorBucket"} + +// A vector bucket resolves its caller the way a table bucket does: an +// identity with an account owns what it created and is bound by a Deny +// naming its account, a federated identity is its issuer-scoped user id, and +// only the admin account's admin bypasses a Deny. +func TestVectorPrincipalMatchesTableBuckets(t *testing.T) { + cases := []struct { + name string + ident *s3api.Identity + principal string + owns bool + denied bool + }{ + {name: "admin", ident: adminIdentity(), principal: "admin", owns: true, denied: false}, + {name: "admin-action", ident: &s3api.Identity{Name: "ops", Account: &s3api.Account{Id: "ops"}, + Actions: []s3api.Action{s3_constants.ACTION_ADMIN}}, principal: "ops", owns: true, denied: true}, + {name: "account", ident: &s3api.Identity{Name: "carol", Account: &s3api.Account{Id: "team-acct"}, + Actions: createActions}, principal: "team-acct", owns: true, denied: true}, + {name: "federated", ident: &s3api.Identity{Name: "alice-sub", Account: &s3api.Account{Id: "alice-sub"}, + Claims: map[string]interface{}{"aws:userid": "uid-alice", "sub": "alice-sub"}, Actions: createActions}, + principal: "uid-alice", owns: true, denied: true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + e := newAuthTestEnv(t) + e.createBuckets(c.ident, c.name) + table, vector := e.canGet(c.ident, c.name) + if table != c.owns || vector != c.owns { + t.Fatalf("creator access: table=%v vector=%v, want %v", table, vector, c.owns) + } + e.deny(c.name, c.principal) + table, vector = e.canGet(c.ident, c.name) + if table != !c.denied || vector != !c.denied { + t.Fatalf("under a deny on %q: table=%v vector=%v, want %v", c.principal, table, vector, !c.denied) + } + }) + } +} + +func (e *authTestEnv) createIndex(bucket string) { + e.t.Helper() + if code := e.vector(adminIdentity(), e.s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: bucket, IndexName: "idx", DataType: "float32", Dimension: 3, DistanceMetric: "euclidean", + }); code != http.StatusOK { + e.t.Fatalf("create index in %s: %d", bucket, code) + } +} + +func (e *authTestEnv) listedVectorBuckets(ident *s3api.Identity) []string { + e.t.Helper() + w := e.vectorResp(ident, e.s.handleListVectorBuckets, &ListVectorBucketsRequest{}) + if w.Code != http.StatusOK { + e.t.Fatalf("list vector buckets: %d %s", w.Code, w.Body.String()) + } + var resp ListVectorBucketsResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + e.t.Fatalf("decode list: %v", err) + } + names := []string{} + for _, b := range resp.VectorBuckets { + names = append(names, b.VectorBucketName) + } + return names +} + +// An identity action scoped to a bucket, s3vectors::, grants that +// operation on that vector bucket and its indexes only, as +// s3tables:: does on a table bucket. A Deny still wins, and the +// scoped form never creates a bucket. +func TestBucketScopedIdentityActions(t *testing.T) { + e := newAuthTestEnv(t) + for _, name := range []string{"team-a", "team-b"} { + e.createBuckets(adminIdentity(), name) + e.createIndex("v-" + name) + } + scoped := &s3api.Identity{Name: "scoped", Account: &s3api.Account{Id: "scoped"}, Actions: []s3api.Action{ + "s3tables:GetTableBucket:t-team-a", "s3tables:CreateTableBucket:t-team-c", + "s3vectors:GetVectorBucket:v-team-a", "s3vectors:GetIndex:v-team-a", "s3vectors:CreateVectorBucket:v-team-c", + }} + + if table, vector := e.canGet(scoped, "team-a"); !table || !vector { + t.Fatalf("granted bucket: table=%v vector=%v, want both", table, vector) + } + if table, vector := e.canGet(scoped, "team-b"); table || vector { + t.Fatalf("other bucket: table=%v vector=%v, want neither", table, vector) + } + getIndex := func(ident *s3api.Identity, bucket string) int { + return e.vector(ident, e.s.handleGetIndex, &GetIndexRequest{VectorBucketName: bucket, IndexName: "idx"}) + } + getVectors := func(ident *s3api.Identity, bucket string) int { + return e.vector(ident, e.s.handleGetVectors, &GetVectorsRequest{VectorBucketName: bucket, IndexName: "idx", Keys: []string{"a"}}) + } + if code := getIndex(scoped, "v-team-a"); code != http.StatusOK { + t.Fatalf("GetIndex in granted bucket: %d", code) + } + if code := getIndex(scoped, "v-team-b"); code != http.StatusForbidden { + t.Fatalf("GetIndex in other bucket: %d, want 403", code) + } + if code := getVectors(scoped, "v-team-a"); code != http.StatusForbidden { + t.Fatalf("ungranted GetVectors in granted bucket: %d, want 403", code) + } + if got := e.listedVectorBuckets(scoped); len(got) != 1 || got[0] != "v-team-a" { + t.Fatalf("listing: %v, want [v-team-a]", got) + } + if err := e.table(scoped, "CreateTableBucket", &s3tables.CreateTableBucketRequest{Name: "t-team-c"}); err == nil { + t.Fatalf("scoped CreateTableBucket created a table bucket") + } + if code := e.vector(scoped, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "v-team-c"}); code != http.StatusForbidden { + t.Fatalf("scoped CreateVectorBucket: %d, want 403", code) + } + + wild := &s3api.Identity{Name: "wild", Account: &s3api.Account{Id: "wild"}, Actions: []s3api.Action{"s3vectors:*:v-team-a"}} + if code := getVectors(wild, "v-team-a"); code != http.StatusOK { + t.Fatalf("wildcard grant in its bucket: %d", code) + } + if code := getIndex(wild, "v-team-b"); code != http.StatusForbidden { + t.Fatalf("wildcard grant in other bucket: %d, want 403", code) + } + if code := e.vector(wild, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "v-team-c"}); code != http.StatusForbidden { + t.Fatalf("wildcard bucket grant created a bucket: %d, want 403", code) + } + + e.deny("team-a", "scoped") + if table, vector := e.canGet(scoped, "team-a"); table || vector { + t.Fatalf("granted bucket under a deny: table=%v vector=%v, want neither", table, vector) + } + if code := getIndex(scoped, "v-team-a"); code != http.StatusForbidden { + t.Fatalf("GetIndex under a deny: %d, want 403", code) + } +} diff --git a/weed/s3api/s3vectors/server.go b/weed/s3api/s3vectors/server.go index af77da9b84..b901881eaf 100644 --- a/weed/s3api/s3vectors/server.go +++ b/weed/s3api/s3vectors/server.go @@ -65,17 +65,28 @@ func (s *Server) SetRegion(region string) { s.region = region } // SetAccountID configures the account id embedded in the ARNs this server mints. func (s *Server) SetAccountID(id string) { s.accountID = id } -// RegisterRoutes mounts POST / dispatching on the X-Amz-Target header and -// POST /{Operation} for clients that route per-path. Metrics wrap Auth like the -// S3 track middleware so rejected requests still count toward the operation. +// SetEncryptionChecker tells CreateVectorBucket whether the gateway can +// apply the AES256 a bucket declares. +func (s *Server) SetEncryptionChecker(fn func() bool) { s.internal.SetEncryptionChecker(fn) } + +// RegisterRoutes mounts POST / dispatching on the X-Amz-Target header, +// POST /{Operation} for clients that route per-path, and the tagging calls on +// /tags/{resourceArn}. Metrics wrap Auth like the S3 track middleware so +// rejected requests still count toward the operation. func (s *Server) RegisterRoutes(router *mux.Router) { - router.HandleFunc("/", s.instrument(s.Auth(s.dispatch))).Methods(http.MethodPost) - router.HandleFunc("/{op}", s.instrument(s.Auth(s.dispatch))).Methods(http.MethodPost) + handler := s.instrument(s.Auth(s.dispatch)) + router.HandleFunc("/", handler).Methods(http.MethodPost) + router.HandleFunc(tagsPathPrefix+"{resourceArn:.+}", handler).Methods(http.MethodGet, http.MethodPost, http.MethodDelete) + router.HandleFunc("/{op}", handler).Methods(http.MethodPost) } -// operationFromRequest resolves the operation the way dispatch does: the last -// path segment, or the X-Amz-Target suffix when the path is bare. +// operationFromRequest resolves the operation the way dispatch does: the +// method on a /tags/ path, else the last path segment, or the X-Amz-Target +// suffix when the path is bare. func operationFromRequest(r *http.Request) string { + if strings.HasPrefix(r.URL.Path, tagsPathPrefix) { + return tagOperation(r.Method) + } op := strings.Trim(r.URL.Path, "/") if i := strings.LastIndex(op, "/"); i >= 0 { op = op[i+1:] @@ -125,8 +136,18 @@ func (s *Server) dispatch(w http.ResponseWriter, r *http.Request) { // Auth authenticates the caller and puts the identity in the request context. func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + if r.ContentLength > maxRequestBodyBytes { + writeError(w, http.StatusBadRequest, "ValidationException", errRequestBodyTooLarge.Error()) + return + } + r = r.WithContext(s3_constants.SetRequestBodyLimitInContext(r.Context(), maxRequestBodyBytes)) identity, identityObj, errCode := s.authenticator.AuthenticateRequest(r) - if errCode != s3err.ErrNone { + switch errCode { + case s3err.ErrNone: + case s3err.ErrEntityTooLarge: + writeError(w, http.StatusBadRequest, "ValidationException", errRequestBodyTooLarge.Error()) + return + default: writeError(w, http.StatusForbidden, "AccessDenied", "authentication failed") return } diff --git a/weed/s3api/s3vectors/tagging.go b/weed/s3api/s3vectors/tagging.go new file mode 100644 index 0000000000..c7370a67f1 --- /dev/null +++ b/weed/s3api/s3vectors/tagging.go @@ -0,0 +1,191 @@ +package s3vectors + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" +) + +// maxResourceTags is how many tags a vector bucket or index may carry. +const maxResourceTags = 50 + +const tagsPathPrefix = "/tags/" + +// tagOperation names the tagging call a /tags/{resourceArn} request makes. +func tagOperation(method string) string { + switch method { + case http.MethodGet: + return "ListTagsForResource" + case http.MethodDelete: + return "UntagResource" + } + return "TagResource" +} + +// readTagRequest decodes the body of a POST; GET and DELETE carry everything +// in the path and query. +func readTagRequest(r *http.Request, req interface{}) error { + if r.Method != http.MethodPost { + return nil + } + return readJSON(r, req) +} + +// tagTarget resolves a tagging call's resourceArn, taken from the path when +// the request was routed there, to the vector bucket or index and the ARN the +// catalog keys that record by. +func (s *Server) tagTarget(r *http.Request, bodyArn string) (vectorBucketARNs, string, error) { + arn := bodyArn + if v := mux.Vars(r)["resourceArn"]; v != "" { + arn = v + } + var v vectorBucketARNs + if arn == "" { + return v, "", fmt.Errorf("resourceArn is required") + } + if !strings.HasPrefix(arn, "arn:") { + return v, "", fmt.Errorf("resourceArn must be an ARN, got %q", arn) + } + if err := s.arnParts(&v, "", arn); err != nil { + return v, "", err + } + catalogArn := s.s3tablesArn(v.bucket) + if v.index != "" { + catalogArn = fmt.Sprintf("%s/table/%s/%s", catalogArn, indexNamespace, v.index) + } + return v, catalogArn, nil +} + +// authorizeTagging grants action on the tagged bucket or index and returns a +// context binding the catalog calls to the bucket it authorized. An index +// target must be a vector index, not another table in its namespace. +func (s *Server) authorizeTagging(r *http.Request, action string, arns vectorBucketARNs) (context.Context, error) { + record, err := s.authorizeVector(r.Context(), r, action, arns.bucket, arns.index) + if err != nil { + return nil, err + } + ctx := s3tables.WithTableBucketCreatedAt(r.Context(), record.CreatedAt) + if arns.index != "" { + index, err := s.indexRecord(ctx, arns.bucket, arns.index) + if err != nil { + return nil, err + } + if format, _ := index["format"].(string); format != s3tables.FormatLance { + return nil, &s3tables.S3TablesError{Type: s3tables.ErrCodeInvalidRequest, + Message: fmt.Sprintf("index %q is not a vector index", arns.index)} + } + } + return ctx, nil +} + +func (s *Server) resourceTags(ctx context.Context, catalogArn string) (map[string]string, error) { + var resp s3tables.ListTagsForResourceResponse + if err := s.internalCatalog(ctx, "ListTagsForResource", + &s3tables.ListTagsForResourceRequest{ResourceARN: catalogArn}, &resp); err != nil { + return nil, err + } + if resp.Tags == nil { + resp.Tags = map[string]string{} + } + return resp.Tags, nil +} + +func (s *Server) handleTagResource(w http.ResponseWriter, r *http.Request) { + var req TagResourceRequest + if err := readTagRequest(r, &req); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + arns, catalogArn, err := s.tagTarget(r, req.ResourceArn) + if err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + if len(req.Tags) == 0 { + writeError(w, http.StatusBadRequest, "ValidationException", "tags are required") + return + } + if err := s3tables.ValidateTagsWithLimit(req.Tags, maxResourceTags); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + ctx, err := s.authorizeTagging(r, "s3vectors:TagResource", arns) + if err != nil { + catalogErr(w, err) + return + } + if err := s.internalCatalog(s3tables.WithMaxTags(ctx, maxResourceTags), "TagResource", + &s3tables.TagResourceRequest{ResourceARN: catalogArn, Tags: req.Tags}, nil); err != nil { + catalogErr(w, err) + return + } + writeJSON(w, http.StatusOK, &TagResourceResponse{}) +} + +func (s *Server) handleUntagResource(w http.ResponseWriter, r *http.Request) { + var req UntagResourceRequest + if err := readTagRequest(r, &req); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + if keys, ok := r.URL.Query()["tagKeys"]; ok { + req.TagKeys = keys + } + arns, catalogArn, err := s.tagTarget(r, req.ResourceArn) + if err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + if len(req.TagKeys) == 0 { + writeError(w, http.StatusBadRequest, "ValidationException", "tagKeys are required") + return + } + ctx, err := s.authorizeTagging(r, "s3vectors:UntagResource", arns) + if err != nil { + catalogErr(w, err) + return + } + key := make(map[string]string, 1) + for _, k := range req.TagKeys { + clear(key) + key[k] = "" + if err := s3tables.ValidateTagsWithLimit(key, 1); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + } + if err := s.internalCatalog(ctx, "UntagResource", + &s3tables.UntagResourceRequest{ResourceARN: catalogArn, TagKeys: req.TagKeys}, nil); err != nil { + catalogErr(w, err) + return + } + writeJSON(w, http.StatusOK, &UntagResourceResponse{}) +} + +func (s *Server) handleListTagsForResource(w http.ResponseWriter, r *http.Request) { + var req ListTagsForResourceRequest + if err := readTagRequest(r, &req); err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + arns, catalogArn, err := s.tagTarget(r, req.ResourceArn) + if err != nil { + writeError(w, http.StatusBadRequest, "ValidationException", err.Error()) + return + } + ctx, err := s.authorizeTagging(r, "s3vectors:ListTagsForResource", arns) + if err != nil { + catalogErr(w, err) + return + } + tags, err := s.resourceTags(ctx, catalogArn) + if err != nil { + catalogErr(w, err) + return + } + writeJSON(w, http.StatusOK, &ListTagsForResourceResponse{Tags: tags}) +} diff --git a/weed/s3api/s3vectors/tagging_test.go b/weed/s3api/s3vectors/tagging_test.go new file mode 100644 index 0000000000..d68eec4896 --- /dev/null +++ b/weed/s3api/s3vectors/tagging_test.go @@ -0,0 +1,355 @@ +package s3vectors + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/s3api" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest" +) + +type tagClient struct { + t *testing.T + ts *httptest.Server + fs *s3tablestest.MemFiler +} + +func newTagClient(t *testing.T) (*Server, *tagClient) { + fs := s3tablestest.Start(t) + s := NewServer(s3tables.NewManagerClient(fs.Client), headerAuth{}, newFakeVectors(0)) + router := mux.NewRouter().SkipClean(true) + s.RegisterRoutes(router) + ts := httptest.NewServer(router) + t.Cleanup(ts.Close) + return s, &tagClient{t: t, ts: ts, fs: fs} +} + +func (c *tagClient) do(identity, method, path string, header http.Header, payload interface{}) (int, string) { + c.t.Helper() + var body io.Reader + if payload != nil { + raw, err := json.Marshal(payload) + if err != nil { + c.t.Fatalf("marshal: %v", err) + } + body = bytes.NewReader(raw) + } + req, err := http.NewRequest(method, c.ts.URL+path, body) + if err != nil { + c.t.Fatalf("build %s %s: %v", method, path, err) + } + for k, v := range header { + req.Header[k] = v + } + if identity != "" { + req.Header.Set("X-Test-Identity", identity) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + c.t.Fatalf("%s %s: %v", method, path, err) + } + defer resp.Body.Close() + out, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(out) +} + +func (c *tagClient) op(identity, op string, payload interface{}) (int, string) { + return c.do(identity, http.MethodPost, "/"+op, nil, payload) +} + +func (c *tagClient) tag(identity, arn string, tags map[string]string) (int, string) { + return c.do(identity, http.MethodPost, tagsPathPrefix+url.PathEscape(arn), nil, map[string]interface{}{"tags": tags}) +} + +func (c *tagClient) untag(identity, arn string, keys ...string) (int, string) { + q := url.Values{"tagKeys": keys} + return c.do(identity, http.MethodDelete, tagsPathPrefix+url.PathEscape(arn)+"?"+q.Encode(), nil, nil) +} + +func (c *tagClient) list(identity, arn string) (int, map[string]string) { + c.t.Helper() + code, body := c.do(identity, http.MethodGet, tagsPathPrefix+url.PathEscape(arn), nil, nil) + if code != http.StatusOK { + return code, nil + } + var resp struct { + Tags map[string]string `json:"tags"` + } + if err := json.Unmarshal([]byte(body), &resp); err != nil || resp.Tags == nil { + c.t.Fatalf("decode ListTagsForResource %q: %v", body, err) + } + return code, resp.Tags +} + +func mustStatus(t *testing.T, what string, code int, body string, want int) { + t.Helper() + if code != want { + t.Fatalf("%s = %d %s, want %d", what, code, body, want) + } +} + +func TestTagVectorBucketsAndIndexes(t *testing.T) { + s, c := newTagClient(t) + code, body := c.op("", "CreateVectorBucket", &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}) + mustStatus(t, "create bucket", code, body, http.StatusOK) + code, body = c.op("", "CreateIndex", &CreateIndexRequest{ + VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, + }) + mustStatus(t, "create index", code, body, http.StatusOK) + bucketArn, indexArn := s.vectorBucketArn("vec-bucket"), s.indexArn("vec-bucket", "idx") + + if _, tags := c.list("", bucketArn); len(tags) != 0 { + t.Fatalf("new bucket tags = %v, want none", tags) + } + code, body = c.tag("", bucketArn, map[string]string{"team": "search", "env": "dev"}) + mustStatus(t, "tag bucket", code, body, http.StatusOK) + code, body = c.tag("", indexArn, map[string]string{"team": "ranking"}) + mustStatus(t, "tag index", code, body, http.StatusOK) + code, body = c.tag("", bucketArn, map[string]string{"env": "prod"}) + mustStatus(t, "retag bucket", code, body, http.StatusOK) + + if _, tags := c.list("", bucketArn); !reflect.DeepEqual(tags, map[string]string{"team": "search", "env": "prod"}) { + t.Fatalf("bucket tags = %v", tags) + } + if _, tags := c.list("", indexArn); !reflect.DeepEqual(tags, map[string]string{"team": "ranking"}) { + t.Fatalf("index tags = %v", tags) + } + + code, body = c.untag("", bucketArn, "env", "absent") + mustStatus(t, "untag bucket", code, body, http.StatusOK) + if _, tags := c.list("", bucketArn); !reflect.DeepEqual(tags, map[string]string{"team": "search"}) { + t.Fatalf("bucket tags after untag = %v", tags) + } + + // Clients that post the operation by name carry resourceArn in the body. + code, body = c.do("", http.MethodPost, "/", http.Header{"X-Amz-Target": {"AmazonS3Vectors.ListTagsForResource"}}, + &ListTagsForResourceRequest{ResourceArn: indexArn}) + mustStatus(t, "list by target header", code, body, http.StatusOK) + var resp ListTagsForResourceResponse + if err := json.Unmarshal([]byte(body), &resp); err != nil || resp.Tags["team"] != "ranking" { + t.Fatalf("list by target header = %s (%v)", body, err) + } +} + +func TestTagResourceValidation(t *testing.T) { + s, c := newTagClient(t) + code, body := c.op("", "CreateVectorBucket", &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}) + mustStatus(t, "create bucket", code, body, http.StatusOK) + bucketArn := s.vectorBucketArn("vec-bucket") + + code, body = c.tag("", s.indexArn("vec-bucket", "missing"), map[string]string{"a": "b"}) + mustStatus(t, "tag missing index", code, body, http.StatusNotFound) + code, _ = c.list("", s.vectorBucketArn("no-such-bucket")) + mustStatus(t, "list missing bucket", code, "", http.StatusNotFound) + code, body = c.tag("", "vec-bucket", map[string]string{"a": "b"}) + mustStatus(t, "tag by name", code, body, http.StatusBadRequest) + code, body = c.tag("", "arn:aws:s3vectors:elsewhere:1:bucket/vec-bucket", map[string]string{"a": "b"}) + mustStatus(t, "tag foreign ARN", code, body, http.StatusBadRequest) + code, body = c.tag("", bucketArn, map[string]string{}) + mustStatus(t, "tag with no tags", code, body, http.StatusBadRequest) + code, body = c.tag("", bucketArn, map[string]string{"bad key!": "v"}) + mustStatus(t, "tag with bad key", code, body, http.StatusBadRequest) + code, body = c.untag("", bucketArn) + mustStatus(t, "untag with no keys", code, body, http.StatusBadRequest) + code, body = c.untag("", bucketArn, "") + mustStatus(t, "untag an empty key", code, body, http.StatusBadRequest) + code, body = c.untag("", bucketArn, strings.Repeat("k", 129)) + mustStatus(t, "untag a key past 128 characters", code, body, http.StatusBadRequest) + + tags := map[string]string{} + for i := 0; i < maxResourceTags; i++ { + tags[fmt.Sprintf("k%d", i)] = "v" + } + code, body = c.tag("", bucketArn, tags) + mustStatus(t, "tag up to the limit", code, body, http.StatusOK) + code, body = c.tag("", bucketArn, map[string]string{"k0": "changed"}) + mustStatus(t, "overwrite at the limit", code, body, http.StatusOK) + code, body = c.tag("", bucketArn, map[string]string{"one-more": "v"}) + mustStatus(t, "tag past the limit", code, body, http.StatusBadRequest) + tags["one-more"] = "v" + code, body = c.tag("", s.vectorBucketArn("vec-bucket"), tags) + mustStatus(t, "request past the limit", code, body, http.StatusBadRequest) + if _, got := c.list("", bucketArn); len(got) != maxResourceTags || got["k0"] != "changed" { + t.Fatalf("tags after refused requests: %d tags, k0=%q", len(got), got["k0"]) + } +} + +func TestTaggingNeedsItsOwnGrant(t *testing.T) { + s, c := newTagClient(t) + code, body := c.op("", "CreateVectorBucket", &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}) + mustStatus(t, "create bucket", code, body, http.StatusOK) + bucketArn := s.vectorBucketArn("vec-bucket") + code, body = c.tag("", bucketArn, map[string]string{"team": "search"}) + mustStatus(t, "owner tags", code, body, http.StatusOK) + + code, _ = c.list("reader", bucketArn) + mustStatus(t, "list without a grant", code, "", http.StatusForbidden) + code, body = c.op("", "PutVectorBucketPolicy", &PutVectorBucketPolicyRequest{ + VectorBucketName: "vec-bucket", + Policy: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*",` + + `"Action":["s3vectors:ListTagsForResource","s3vectors:GetVectorBucket"],"Resource":"` + bucketArn + `"}]}`, + }) + mustStatus(t, "put policy", code, body, http.StatusOK) + if code, tags := c.list("reader", bucketArn); code != http.StatusOK || tags["team"] != "search" { + t.Fatalf("list under grant = %d %v", code, tags) + } + code, body = c.tag("reader", bucketArn, map[string]string{"team": "other"}) + mustStatus(t, "tag under list grant", code, body, http.StatusForbidden) + code, body = c.untag("reader", bucketArn, "team") + mustStatus(t, "untag under list grant", code, body, http.StatusForbidden) + code, body = c.untag("reader", bucketArn, "") + mustStatus(t, "untag an empty key under list grant", code, body, http.StatusForbidden) + if _, tags := c.list("", bucketArn); tags["team"] != "search" { + t.Fatalf("refused writes changed tags: %v", tags) + } +} + +func TestCreateWithTagsNeedsTagResource(t *testing.T) { + e := newAuthTestEnv(t) + tags := map[string]string{"team": "search"} + listed := func(arn string) map[string]string { + t.Helper() + w := e.vectorResp(adminIdentity(), e.s.handleListTagsForResource, &ListTagsForResourceRequest{ResourceArn: arn}) + var resp ListTagsForResourceResponse + if w.Code != http.StatusOK || json.Unmarshal(w.Body.Bytes(), &resp) != nil { + t.Fatalf("list tags of %s: %d %s", arn, w.Code, w.Body.String()) + } + return resp.Tags + } + creator := func(actions ...s3api.Action) *s3api.Identity { + return &s3api.Identity{Name: "creator", Account: &s3api.Account{Id: "creator"}, Actions: actions} + } + createOnly := creator("s3vectors:CreateVectorBucket", "s3vectors:CreateIndex") + withTag := creator("s3vectors:CreateVectorBucket", "s3vectors:CreateIndex", "s3vectors:TagResource") + + if code := e.vector(createOnly, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "denied", Tags: tags}); code != http.StatusForbidden { + t.Fatalf("tagged bucket create without TagResource: %d, want 403", code) + } + if code := e.vector(adminIdentity(), e.s.handleGetVectorBucket, &GetVectorBucketRequest{VectorBucketName: "denied"}); code != http.StatusNotFound { + t.Fatalf("refused create left a bucket: %d", code) + } + if code := e.vector(withTag, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "vec-bucket", Tags: tags}); code != http.StatusOK { + t.Fatalf("tagged bucket create: %d", code) + } + if got := listed(e.s.vectorBucketArn("vec-bucket")); !reflect.DeepEqual(got, tags) { + t.Fatalf("bucket tags = %v, want %v", got, tags) + } + + e.createBuckets(adminIdentity(), "shared") + index := func(ident *s3api.Identity, name string, tags map[string]string) int { + return e.vector(ident, e.s.handleCreateIndex, &CreateIndexRequest{ + VectorBucketName: "v-shared", IndexName: name, DataType: "float32", Dimension: 3, Tags: tags, + }) + } + if code := index(createOnly, "denied", tags); code != http.StatusForbidden { + t.Fatalf("tagged index create without TagResource: %d, want 403", code) + } + if code := index(createOnly, "plain", nil); code != http.StatusOK { + t.Fatalf("untagged index create: %d", code) + } + if code := index(withTag, "tagged", tags); code != http.StatusOK { + t.Fatalf("tagged index create: %d", code) + } + if got := listed(e.s.indexArn("v-shared", "tagged")); !reflect.DeepEqual(got, tags) { + t.Fatalf("index tags = %v, want %v", got, tags) + } + if got := listed(e.s.indexArn("v-shared", "plain")); len(got) != 0 { + t.Fatalf("untagged index tags = %v", got) + } + + scoped := creator("s3vectors:CreateVectorBucket", "s3vectors:TagResource:team-bucket") + if code := e.vector(scoped, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "other-bucket", Tags: tags}); code != http.StatusForbidden { + t.Fatalf("tagged create outside the scoped TagResource grant: %d, want 403", code) + } + if code := e.vector(scoped, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "team-bucket", Tags: tags}); code != http.StatusOK { + t.Fatalf("tagged create under a TagResource grant scoped to the bucket: %d", code) + } + + tooMany := map[string]string{} + for i := 0; i <= maxResourceTags; i++ { + tooMany[fmt.Sprintf("k%d", i)] = "v" + } + if code := e.vector(withTag, e.s.handleCreateVectorBucket, &CreateVectorBucketRequest{VectorBucketName: "too-many", Tags: tooMany}); code != http.StatusBadRequest { + t.Fatalf("bucket create past the tag limit: %d, want 400", code) + } + if code := index(withTag, "too-many", tooMany); code != http.StatusBadRequest { + t.Fatalf("index create past the tag limit: %d, want 400", code) + } +} + +func TestTaggingRefusesTablesThatAreNotIndexes(t *testing.T) { + s, c := newTagClient(t) + code, body := c.op("", "CreateVectorBucket", &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}) + mustStatus(t, "create bucket", code, body, http.StatusOK) + code, body = c.op("", "CreateIndex", &CreateIndexRequest{ + VectorBucketName: "vec-bucket", IndexName: "idx", DataType: "float32", Dimension: 3, + }) + mustStatus(t, "create index", code, body, http.StatusOK) + indexArn := s.indexArn("vec-bucket", "idx") + code, body = c.tag("", indexArn, map[string]string{"team": "search"}) + mustStatus(t, "tag index", code, body, http.StatusOK) + + dir, name := splitEntryPath(s3tables.GetTablePath("vec-bucket", indexNamespace, "idx")) + entry := c.fs.Get(dir, name) + var meta map[string]interface{} + if err := json.Unmarshal(entry.Extended[s3tables.ExtendedKeyMetadata], &meta); err != nil { + t.Fatalf("decode index metadata: %v", err) + } + meta["format"] = s3tables.FormatIceberg + data, _ := json.Marshal(meta) + entry.Extended[s3tables.ExtendedKeyMetadata] = data + c.fs.Put(dir, name, entry.Extended) + + code, _ = c.list("", indexArn) + mustStatus(t, "list tags of an Iceberg table", code, "", http.StatusBadRequest) + code, body = c.tag("", indexArn, map[string]string{"team": "other"}) + mustStatus(t, "tag an Iceberg table", code, body, http.StatusBadRequest) + code, body = c.untag("", indexArn, "team") + mustStatus(t, "untag an Iceberg table", code, body, http.StatusBadRequest) + if got := string(c.fs.Get(dir, name).Extended[s3tables.ExtendedKeyTags]); got != `{"team":"search"}` { + t.Fatalf("refused calls changed the table's tags: %s", got) + } +} + +func TestUntagManyKeysAllocatesLikeDecoding(t *testing.T) { + s, c := newTagClient(t) + code, out := c.op("", "CreateVectorBucket", &CreateVectorBucketRequest{VectorBucketName: "vec-bucket"}) + mustStatus(t, "create bucket", code, out, http.StatusOK) + body, err := json.Marshal(&UntagResourceRequest{ResourceArn: s.vectorBucketArn("vec-bucket"), TagKeys: make([]string, 1<<18)}) + if err != nil { + t.Fatal(err) + } + allocated := func(f func()) uint64 { + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + f() + runtime.ReadMemStats(&after) + return after.TotalAlloc - before.TotalAlloc + } + decode := allocated(func() { + var req UntagResourceRequest + if err := json.Unmarshal(body, &req); err != nil { + t.Fatal(err) + } + }) + w := httptest.NewRecorder() + handle := allocated(func() { + s.handleUntagResource(w, httptest.NewRequest(http.MethodPost, "/UntagResource", bytes.NewReader(body))) + }) + mustStatus(t, "untag empty keys", w.Code, w.Body.String(), http.StatusBadRequest) + if handle > decode*3/2 { + t.Fatalf("untag allocated %d bytes, decoding the request takes %d", handle, decode) + } +} diff --git a/weed/s3api/s3vectors/types.go b/weed/s3api/s3vectors/types.go index a9c5b96c51..4f5f1496ec 100644 --- a/weed/s3api/s3vectors/types.go +++ b/weed/s3api/s3vectors/types.go @@ -1,8 +1,10 @@ package s3vectors import ( + "bytes" "context" "encoding/json" + "fmt" "io" "net/http" "time" @@ -35,10 +37,19 @@ func writeError(w http.ResponseWriter, status int, code, message string) { }) } -const maxRequestBodyBytes = 32 << 20 +const maxRequestBodyBytes = s3_constants.S3VectorsMaxRequestBodyBytes + +var errRequestBodyTooLarge = fmt.Errorf("request body exceeds %d MiB", maxRequestBodyBytes>>20) func readJSON(r *http.Request, req interface{}) error { - return json.NewDecoder(io.LimitReader(r.Body, maxRequestBodyBytes)).Decode(req) + body, err := io.ReadAll(io.LimitReader(r.Body, maxRequestBodyBytes+1)) + if err != nil { + return err + } + if len(body) > maxRequestBodyBytes { + return errRequestBodyTooLarge + } + return json.NewDecoder(bytes.NewReader(body)).Decode(req) } // Request/response shapes for the AWS S3 Vectors API (awsJson1_1). @@ -51,7 +62,7 @@ type vectorBucketARNs struct { type CreateVectorBucketRequest struct { VectorBucketName string `json:"vectorBucketName"` EncryptionConfiguration map[string]interface{} `json:"encryptionConfiguration,omitempty"` - MetadataResourceTags map[string]string `json:"-"` + Tags map[string]string `json:"tags,omitempty"` } type CreateVectorBucketResponse struct { @@ -125,6 +136,7 @@ type CreateIndexRequest struct { MetadataConfiguration *MetadataConfiguration `json:"metadataConfiguration,omitempty"` VectorBucketName string `json:"vectorBucketName"` VectorBucketArn string `json:"vectorBucketArn"` + Tags map[string]string `json:"tags,omitempty"` } type MetadataConfiguration struct { @@ -284,3 +296,25 @@ type QueryVectorsResponse struct { DistanceMetric string `json:"distanceMetric"` Vectors []*QueryOutputVector `json:"vectors"` } + +type TagResourceRequest struct { + ResourceArn string `json:"resourceArn"` + Tags map[string]string `json:"tags"` +} + +type TagResourceResponse struct{} + +type UntagResourceRequest struct { + ResourceArn string `json:"resourceArn"` + TagKeys []string `json:"tagKeys"` +} + +type UntagResourceResponse struct{} + +type ListTagsForResourceRequest struct { + ResourceArn string `json:"resourceArn"` +} + +type ListTagsForResourceResponse struct { + Tags map[string]string `json:"tags"` +} diff --git a/weed/security/grpc_peer.go b/weed/security/grpc_peer.go index a6b1fba95e..9a2844067d 100644 --- a/weed/security/grpc_peer.go +++ b/weed/security/grpc_peer.go @@ -38,16 +38,26 @@ func CheckGrpcAdminPeer(ctx context.Context, signingKey SigningKey) error { if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") || parts[1] == "" { return status.Error(codes.Unauthenticated, "authorization header must use Bearer scheme") } - parsed, err := DecodeJwt(signingKey, EncodedJwt(parts[1]), &SeaweedFilerAdminClaims{}) - if err != nil || parsed == nil || !parsed.Valid { - return status.Error(codes.Unauthenticated, "invalid admin token") - } - if claims, ok := parsed.Claims.(*SeaweedFilerAdminClaims); !ok || claims.SessionId != "" { + if !IsFilerAdminToken(signingKey, EncodedJwt(parts[1])) { return status.Error(codes.Unauthenticated, "invalid admin token") } return nil } +// IsFilerAdminToken reports whether token is an admin token signed by +// signingKey. An STS session token signed with the same key does not count. +func IsFilerAdminToken(signingKey SigningKey, token EncodedJwt) bool { + if len(signingKey) == 0 || token == "" { + return false + } + parsed, err := DecodeJwt(signingKey, token, &SeaweedFilerAdminClaims{}) + if err != nil || parsed == nil || !parsed.Valid { + return false + } + claims, ok := parsed.Claims.(*SeaweedFilerAdminClaims) + return ok && claims.SessionId == "" +} + // GrpcAdminPeerCredentials returns outgoing metadata carrying a Bearer token // for admin-plane gRPC handlers, or nil when no signing key is configured. func GrpcAdminPeerCredentials(signingKey SigningKey, expiresAfterSec int) metadata.MD { diff --git a/weed/server/filer_grpc_server_create_route_test.go b/weed/server/filer_grpc_server_create_route_test.go index 0c266c5397..8bb4bbbc33 100644 --- a/weed/server/filer_grpc_server_create_route_test.go +++ b/weed/server/filer_grpc_server_create_route_test.go @@ -232,3 +232,40 @@ func TestCreateEntryFailsWhenPriorOwnerIsUnreachable(t *testing.T) { t.Fatal("a create routed away must not also write locally") } } + +func TestCreateEntryRejectsDotSegments(t *testing.T) { + fs, store := createRouteServer(t) + + for _, tc := range []struct{ dir, name string }{ + {"/../escape", "marker.txt"}, + {"/test/./sub", "marker.txt"}, + {"/test", ".."}, + } { + req := createReq(tc.name, false) + req.Directory = tc.dir + resp, err := fs.CreateEntry(context.Background(), req) + if err == nil && resp.Error == "" { + t.Fatalf("CreateEntry %s/%s must be rejected", tc.dir, tc.name) + } + if _, found := store.entries[string(util.NewFullPath(tc.dir, tc.name))]; found { + t.Fatalf("CreateEntry %s/%s stored the raw path", tc.dir, tc.name) + } + } + + resp, err := fs.CreateEntry(context.Background(), createReq("marker.txt", false)) + if err != nil || resp.Error != "" { + t.Fatalf("CreateEntry /test/marker.txt: err=%v resp=%v", err, resp) + } + + update := &filer_pb.UpdateEntryRequest{ + Directory: "/x/../test", + Entry: createReq("marker.txt", false).Entry, + } + update.Entry.Attributes.Mtime++ + if _, err := fs.UpdateEntry(context.Background(), update); err == nil { + t.Fatal("UpdateEntry with a dot segment must be rejected") + } + if _, found := store.entries["/x/../test/marker.txt"]; found { + t.Fatal("UpdateEntry stored the raw path") + } +} diff --git a/weed/server/filer_grpc_server_rename_test.go b/weed/server/filer_grpc_server_rename_test.go index 10d6941406..b793025472 100644 --- a/weed/server/filer_grpc_server_rename_test.go +++ b/weed/server/filer_grpc_server_rename_test.go @@ -559,3 +559,32 @@ func TestAtomicRenameReleasesQueuedStampsOnCommitFailure(t *testing.T) { t.Fatalf("LocalDeliveredThroughTsNs = %d, want %d: a leaked stamp still caps the claim", got, now) } } + +func TestAtomicRenameEntryRejectsDotSegmentTarget(t *testing.T) { + for _, tc := range []struct{ dir, name string }{ + {"/../escape", "dst.txt"}, + {"/", ".."}, + } { + store := newRenameTestStore() + store.entries["/src.txt"] = newFileEntry("/src.txt", 101) + f := newRenameTestFiler(t, store) + f.DirBucketsPath = "/buckets" + server := &FilerServer{filer: f, entryLockTable: util.NewLockTable[util.FullPath]()} + + _, err := server.AtomicRenameEntry(context.Background(), &filer_pb.AtomicRenameEntryRequest{ + OldDirectory: "/", + OldName: "src.txt", + NewDirectory: tc.dir, + NewName: tc.name, + }) + if err == nil { + t.Fatalf("rename to %s/%s must be rejected", tc.dir, tc.name) + } + if _, found := store.entries["/src.txt"]; !found { + t.Fatalf("rename to %s/%s removed the source", tc.dir, tc.name) + } + if len(store.entries) != 1 { + t.Fatalf("rename to %s/%s left entries %v", tc.dir, tc.name, store.entries) + } + } +} diff --git a/weed/server/filer_jwt_test.go b/weed/server/filer_jwt_test.go index 74f5bb9bcc..efd4b99c60 100644 --- a/weed/server/filer_jwt_test.go +++ b/weed/server/filer_jwt_test.go @@ -38,6 +38,7 @@ func TestFilerServer_maybeCheckJwtAuthorization_Scoped(t *testing.T) { token string method string path string + header map[string]string isWrite bool expectAuthorized bool }{ @@ -225,6 +226,63 @@ func TestFilerServer_maybeCheckJwtAuthorization_Scoped(t *testing.T) { isWrite: true, expectAuthorized: false, }, + { + name: "write at the prefix itself allowed", + token: genToken([]string{"/tenant1"}, nil), + method: "POST", + path: "/tenant1", + isWrite: true, + expectAuthorized: true, + }, + { + name: "extract into the prefix directory allowed", + token: genToken([]string{"/tenant1"}, nil), + method: "POST", + path: "/tenant1/?extract=zip", + isWrite: true, + expectAuthorized: true, + }, + { + name: "extract beside an archive inside the prefix allowed", + token: genToken([]string{"/tenant1"}, nil), + method: "PUT", + path: "/tenant1/pack.zip?extract=zip", + isWrite: true, + expectAuthorized: true, + }, + { + name: "extract at the prefix itself writes to its parent, denied", + token: genToken([]string{"/tenant1"}, nil), + method: "POST", + path: "/tenant1?extract=zip", + isWrite: true, + expectAuthorized: false, + }, + { + name: "extract header at the prefix itself denied", + token: genToken([]string{"/tenant1/"}, []string{"PUT"}), + method: "PUT", + path: "/tenant1", + header: map[string]string{"X-Seaweedfs-Extract": "zip"}, + isWrite: true, + expectAuthorized: false, + }, + { + name: "extract at a nested prefix into the sibling parent denied", + token: genToken([]string{"/buckets/bucket-a"}, nil), + method: "POST", + path: "/buckets/bucket-a?extract=zip", + isWrite: true, + expectAuthorized: false, + }, + { + name: "extract with a root prefix allowed", + token: genToken([]string{"/"}, nil), + method: "POST", + path: "/tenant1?extract=zip", + isWrite: true, + expectAuthorized: true, + }, } for _, tt := range tests { @@ -233,6 +291,9 @@ func TestFilerServer_maybeCheckJwtAuthorization_Scoped(t *testing.T) { if tt.token != "" { req.Header.Set("Authorization", "Bearer "+tt.token) } + for k, v := range tt.header { + req.Header.Set(k, v) + } if authorized := fs.maybeCheckJwtAuthorization(req, tt.isWrite); authorized != tt.expectAuthorized { t.Errorf("expected authorized=%v, got %v", tt.expectAuthorized, authorized) } diff --git a/weed/server/filer_server_handlers.go b/weed/server/filer_server_handlers.go index 6fd9bfbf3e..4bdd5d87f9 100644 --- a/weed/server/filer_server_handlers.go +++ b/weed/server/filer_server_handlers.go @@ -308,10 +308,14 @@ func authorizeFilerJwtPaths(r *http.Request, claims *security.SeaweedFilerClaims } // jwtScopedRequestPaths returns every filer path a request touches that must be -// covered by the JWT AllowedPrefixes: the write target (r.URL.Path) plus any -// copy/move source named by the cp.from / mv.from query parameters. +// covered by the JWT AllowedPrefixes: the write target (r.URL.Path), the +// directory a zip extract writes under, and any copy/move source named by the +// cp.from / mv.from query parameters. func jwtScopedRequestPaths(r *http.Request) []string { paths := []string{r.URL.Path} + if (r.Method == http.MethodPost || r.Method == http.MethodPut) && isExtractZip(r) { + paths = append(paths, extractZipRoot(r)+"/") + } if query := r.URL.Query(); query.Has("cp.from") || query.Has("mv.from") { if from := query.Get("cp.from"); from != "" { paths = append(paths, from) diff --git a/weed/server/filer_server_handlers_extract.go b/weed/server/filer_server_handlers_extract.go index c51d7b9c6f..97057ee3fc 100644 --- a/weed/server/filer_server_handlers_extract.go +++ b/weed/server/filer_server_handlers_extract.go @@ -35,6 +35,17 @@ func isExtractZip(r *http.Request) bool { r.Header.Get(s3_constants.SeaweedFSExtract) == "zip" } +// extractZipRoot is the directory members land under: the request path when it +// ends in "/", otherwise its parent, so the path can name the archive. It is +// cleaned so the JWT scope check and the writes agree on dot segments. +func extractZipRoot(r *http.Request) string { + root := r.URL.Path + if !strings.HasSuffix(root, "/") { + root = path.Dir(root) + } + return strings.TrimSuffix(path.Clean(root), "/") +} + // extractZipArchive writes a posted archive's members as individual entries // under the target directory, like AWS Snowball's auto-extract. The archive // itself is only stored when ?extract.keep=true. @@ -81,11 +92,7 @@ func (fs *FilerServer) extractZipArchive(ctx context.Context, w http.ResponseWri return } - root := r.URL.Path - if !strings.HasSuffix(root, "/") { - root = path.Dir(root) - } - root = strings.TrimSuffix(root, "/") + root := extractZipRoot(r) query := r.URL.Query() chunkSize := fs.requestChunkSize(query) @@ -153,7 +160,7 @@ func (fs *FilerServer) extractZipArchive(ctx context.Context, w http.ResponseWri } } - if query.Get("extract.keep") == "true" && archiveName != "." && archiveName != "/" { + if query.Get("extract.keep") == "true" && archiveName != "." && archiveName != ".." && archiveName != "/" { mode := os.FileMode(0660) if modeStr := query.Get("mode"); modeStr != "" { if parsed, parseErr := strconv.ParseUint(modeStr, 8, 32); parseErr == nil { diff --git a/weed/server/filer_server_handlers_extract_test.go b/weed/server/filer_server_handlers_extract_test.go index 9461e79e5a..3040a9ad4a 100644 --- a/weed/server/filer_server_handlers_extract_test.go +++ b/weed/server/filer_server_handlers_extract_test.go @@ -168,6 +168,28 @@ func TestExtractZipArchiveKeep(t *testing.T) { } } +func TestExtractZipArchiveKeepSkipsParentName(t *testing.T) { + fs, f := newExtractTestServer(t) + zipBytes := buildTestZip(t, map[string]string{"x.txt": "data"}, nil, nil) + + var body bytes.Buffer + mw := multipartWriter(t, &body, "..", zipBytes) + r := httptest.NewRequest(http.MethodPost, "/a/b/?extract=zip&extract.keep=true", &body) + r.Header.Set("Content-Type", mw) + w := httptest.NewRecorder() + fs.PostHandler(w, r, int64(body.Len())) + + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, body: %s", w.Code, w.Body.String()) + } + if _, err := f.FindEntry(r.Context(), util.FullPath("/a/b/..")); err == nil { + t.Fatal("archive kept under a parent-directory name") + } + if _, err := f.FindEntry(r.Context(), util.FullPath("/a/b/x.txt")); err != nil { + t.Fatalf("member missing: %v", err) + } +} + func TestExtractZipArchiveNotAZip(t *testing.T) { fs, _ := newExtractTestServer(t) r := httptest.NewRequest(http.MethodPut, "/dst/notzip.bin?extract=zip", bytes.NewReader([]byte("not a zip"))) @@ -231,6 +253,28 @@ func TestExtractZipArchiveRawPost(t *testing.T) { } } +func TestExtractZipArchiveCleansDotSegments(t *testing.T) { + fs, f := newExtractTestServer(t) + zipBytes := buildTestZip(t, map[string]string{"raw.txt": "raw"}, nil, nil) + + // A percent-encoded ".." survives path cleaning in front of the mux, so the + // handler sees it decoded and must land members under the cleaned root. + r := httptest.NewRequest(http.MethodPost, "/other/%2e%2e/dst/?extract=zip", bytes.NewReader(zipBytes)) + r.Header.Set("Content-Type", "application/zip") + w := httptest.NewRecorder() + fs.PostHandler(w, r, int64(len(zipBytes))) + + if w.Code != http.StatusCreated { + t.Fatalf("status = %d, body: %s", w.Code, w.Body.String()) + } + if _, err := f.FindEntry(r.Context(), util.FullPath("/dst/raw.txt")); err != nil { + t.Fatalf("member missing under the cleaned root: %v", err) + } + if _, err := f.FindEntry(r.Context(), util.FullPath("/other/../dst/raw.txt")); err == nil { + t.Fatal("member written under the raw root") + } +} + func TestExtractZipArchiveMultipartPost(t *testing.T) { fs, f := newExtractTestServer(t) zipBytes := buildTestZip(t, map[string]string{"img/1.png": "png-bytes"}, nil, nil) diff --git a/weed/shell/command_ec_common.go b/weed/shell/command_ec_common.go index ea7a9fd9e2..bf49b02b7c 100644 --- a/weed/shell/command_ec_common.go +++ b/weed/shell/command_ec_common.go @@ -93,10 +93,13 @@ func collectTopologyInfo(commandEnv *CommandEnv, delayBeforeCollecting time.Dura if delayBeforeCollecting > 0 { time.Sleep(delayBeforeCollecting) } + return fetchTopologyInfo(context.Background(), commandEnv, &master_pb.VolumeListRequest{}) +} +func fetchTopologyInfo(ctx context.Context, commandEnv *CommandEnv, request *master_pb.VolumeListRequest) (topoInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, err error) { var resp *master_pb.VolumeListResponse - err = commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error { - resp, err = pb.CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{}) + err = commandEnv.MasterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error { + resp, err = pb.CollectVolumeList(ctx, client, request) return err }) if err != nil { @@ -107,22 +110,88 @@ func collectTopologyInfo(commandEnv *CommandEnv, delayBeforeCollecting time.Dura } func collectDataNodes(commandEnv *CommandEnv, delayBeforeCollecting time.Duration) ([]*master_pb.DataNodeInfo, error) { - dataNodes := []*master_pb.DataNodeInfo{} - topo, _, err := collectTopologyInfo(commandEnv, delayBeforeCollecting) if err != nil { return nil, err } + return topologyDataNodes(topo), nil +} +func topologyDataNodes(topo *master_pb.TopologyInfo) []*master_pb.DataNodeInfo { + dataNodes := []*master_pb.DataNodeInfo{} for _, dci := range topo.GetDataCenterInfos() { for _, r := range dci.GetRackInfos() { - for _, dn := range r.GetDataNodeInfos() { - dataNodes = append(dataNodes, dn) - } + dataNodes = append(dataNodes, r.GetDataNodeInfos()...) } } + return dataNodes +} - return dataNodes, nil +// collectVolumeServerAddresses resolves nodes with volumeServerAddresses from +// a topology-only listing. An explicit node list bounds the lookup and is used +// as given when the topology is unavailable, so it still works while the +// master is down. +func collectVolumeServerAddresses(commandEnv *CommandEnv, nodes string) ([]pb.ServerAddress, error) { + if nodes != "" && everyNodeNamesGrpcPort(nodes) { + return volumeServerAddresses(nil, nodes), nil + } + ctx := context.Background() + if nodes != "" { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 30*time.Second) + defer cancel() + } + topo, _, err := fetchTopologyInfo(ctx, commandEnv, &master_pb.VolumeListRequest{TopologyOnly: true}) + if err != nil { + if nodes == "" { + return nil, err + } + glog.V(0).Infof("dialing volume servers as given, topology unavailable: %v", err) + } + return volumeServerAddresses(topologyDataNodes(topo), nodes), nil +} + +func everyNodeNamesGrpcPort(nodes string) bool { + for _, node := range strings.Split(nodes, ",") { + node = strings.TrimSpace(node) + if node == "" { + continue + } + portsSepIndex := strings.LastIndex(node, ":") + if portsSepIndex < 0 || !strings.Contains(node[portsSepIndex+1:], ".") { + return false + } + } + return true +} + +// volumeServerAddresses returns the comma-separated nodes, or every data node +// when nodes is empty, dialing each registered server on its gRPC port unless +// the node names one. +func volumeServerAddresses(dataNodes []*master_pb.DataNodeInfo, nodes string) []pb.ServerAddress { + registered := make(map[string]pb.ServerAddress, len(dataNodes)) + all := make([]pb.ServerAddress, 0, len(dataNodes)) + for _, dn := range dataNodes { + addr := pb.NewServerAddressFromDataNode(dn) + registered[addr.ToHttpAddress()] = addr + all = append(all, addr) + } + if nodes == "" { + return all + } + addrs := []pb.ServerAddress{} + for _, node := range strings.Split(nodes, ",") { + node = strings.TrimSpace(node) + if node == "" { + continue + } + if addr, found := registered[node]; found { + addrs = append(addrs, addr) + } else { + addrs = append(addrs, pb.ServerAddress(node)) + } + } + return addrs } func collectEcNodesForDC(commandEnv *CommandEnv, selectedDataCenter string, diskType types.DiskType) (ecNodes []*EcNode, totalFreeEcSlots int, err error) { diff --git a/weed/shell/command_ec_scrub.go b/weed/shell/command_ec_scrub.go index 13414fba3e..00a1fa072d 100644 --- a/weed/shell/command_ec_scrub.go +++ b/weed/shell/command_ec_scrub.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/seaweedfs/seaweedfs/weed/ec" - "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" ) @@ -53,23 +52,9 @@ func (c *commandEcVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer return } - volumeServerAddrs := []pb.ServerAddress{} - if *nodesStr != "" { - for _, addr := range strings.Split(*nodesStr, ",") { - addr = strings.TrimSpace(addr) - if addr == "" { - continue - } - volumeServerAddrs = append(volumeServerAddrs, pb.ServerAddress(addr)) - } - } else { - dns, err := collectDataNodes(commandEnv, 0) - if err != nil { - return err - } - for _, dn := range dns { - volumeServerAddrs = append(volumeServerAddrs, pb.ServerAddress(dn.Address)) - } + volumeServerAddrs, err := collectVolumeServerAddresses(commandEnv, *nodesStr) + if err != nil { + return err } volumeIDs := []uint32{} diff --git a/weed/shell/command_volume_scrub.go b/weed/shell/command_volume_scrub.go index 95afd3a329..0671075eea 100644 --- a/weed/shell/command_volume_scrub.go +++ b/weed/shell/command_volume_scrub.go @@ -62,19 +62,8 @@ func (c *commandVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer io return } - c.volumeServerAddrs = []pb.ServerAddress{} - if *nodesStr != "" { - for _, addr := range strings.Split(*nodesStr, ",") { - c.volumeServerAddrs = append(c.volumeServerAddrs, pb.ServerAddress(addr)) - } - } else { - dns, err := collectDataNodes(commandEnv, 0) - if err != nil { - return err - } - for _, dn := range dns { - c.volumeServerAddrs = append(c.volumeServerAddrs, pb.ServerAddress(dn.Address)) - } + if c.volumeServerAddrs, err = collectVolumeServerAddresses(commandEnv, *nodesStr); err != nil { + return err } c.volumeIDs = []uint32{} diff --git a/weed/shell/command_volume_scrub_test.go b/weed/shell/command_volume_scrub_test.go new file mode 100644 index 0000000000..8a2d29d867 --- /dev/null +++ b/weed/shell/command_volume_scrub_test.go @@ -0,0 +1,54 @@ +package shell + +import ( + "reflect" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" +) + +func TestVolumeServerAddressesUseRegisteredGrpcPort(t *testing.T) { + dataNodes := []*master_pb.DataNodeInfo{ + {Id: "10.0.0.1:8080", Address: "10.0.0.1:8080", GrpcPort: 18080}, + {Id: "10.0.0.2:7000", Address: "10.0.0.2:7000", GrpcPort: 7500}, + {Id: "10.0.0.3:8081"}, + } + tests := []struct { + nodes string + wantGrpc []string + }{ + {"", []string{"10.0.0.1:18080", "10.0.0.2:7500", "10.0.0.3:18081"}}, + {"10.0.0.2:7000", []string{"10.0.0.2:7500"}}, + {" 10.0.0.2:7000 ,,10.0.0.1:8080", []string{"10.0.0.2:7500", "10.0.0.1:18080"}}, + {"10.0.0.9:7000.7600", []string{"10.0.0.9:7600"}}, + {"10.0.0.2:7000.7600", []string{"10.0.0.2:7600"}}, + } + for _, tt := range tests { + var gotGrpc []string + for _, addr := range volumeServerAddresses(dataNodes, tt.nodes) { + gotGrpc = append(gotGrpc, addr.ToGrpcAddress()) + } + if !reflect.DeepEqual(gotGrpc, tt.wantGrpc) { + t.Errorf("volumeServerAddresses(%q) dials %v, want %v", tt.nodes, gotGrpc, tt.wantGrpc) + } + } +} + +func TestEveryNodeNamesGrpcPort(t *testing.T) { + tests := []struct { + nodes string + want bool + }{ + {"10.0.0.9:7000.7600", true}, + {" 10.0.0.9:7000.7600 ,,[::1]:8080.18080", true}, + {"10.0.0.9:7000.7600,10.0.0.2:7000", false}, + {"10.0.0.2:7000", false}, + {"volume.example.com", false}, + {"volume.example.com:8080", false}, + } + for _, tt := range tests { + if got := everyNodeNamesGrpcPort(tt.nodes); got != tt.want { + t.Errorf("everyNodeNamesGrpcPort(%q) = %v, want %v", tt.nodes, got, tt.want) + } + } +} diff --git a/weed/shell/command_volume_server_state.go b/weed/shell/command_volume_server_state.go index 9ab850f568..03fd5814f0 100644 --- a/weed/shell/command_volume_server_state.go +++ b/weed/shell/command_volume_server_state.go @@ -86,19 +86,9 @@ func (c *commandVolumeServerState) Do(args []string, commandEnv *CommandEnv, wri return fmt.Errorf("--maintenanceOn and --maintenanceOff are mutually exclusive") } - volumeServerAddrs := []pb.ServerAddress{} - if *nodesStr != "" { - for _, addr := range strings.Split(*nodesStr, ",") { - volumeServerAddrs = append(volumeServerAddrs, pb.ServerAddress(addr)) - } - } else { - dns, err := collectDataNodes(commandEnv, 0) - if err != nil { - return err - } - for _, dn := range dns { - volumeServerAddrs = append(volumeServerAddrs, pb.ServerAddress(dn.Address)) - } + volumeServerAddrs, err := collectVolumeServerAddresses(commandEnv, *nodesStr) + if err != nil { + return err } slices.Sort(volumeServerAddrs) if len(volumeServerAddrs) == 0 { diff --git a/weed/storage/backend/s3_backend/s3_backend.go b/weed/storage/backend/s3_backend/s3_backend.go index 08651dedea..689286ad0c 100644 --- a/weed/storage/backend/s3_backend/s3_backend.go +++ b/weed/storage/backend/s3_backend/s3_backend.go @@ -1,6 +1,7 @@ package s3_backend import ( + "context" "fmt" "io" "math" @@ -222,7 +223,9 @@ func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n bytesRange := fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))-1) - getObjectOutput, getObjectErr := s3backendStorageFile.backendStorage.conn.GetObject(&s3.GetObjectInput{ + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + getObjectOutput, getObjectErr := s3backendStorageFile.backendStorage.conn.GetObjectWithContext(ctx, &s3.GetObjectInput{ Bucket: &s3backendStorageFile.backendStorage.bucket, Key: &s3backendStorageFile.key, Range: &bytesRange, @@ -232,6 +235,8 @@ func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n return 0, fmt.Errorf("bucket %s GetObject %s: %v", s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, getObjectErr) } defer getObjectOutput.Body.Close() + stalled := time.AfterFunc(s3ReadTimeout, cancel) + defer stalled.Stop() // glog.V(3).Infof("read %s %s", s3backendStorageFile.key, bytesRange) // glog.V(3).Infof("content range: %s, contentLength: %d", *getObjectOutput.ContentRange, *getObjectOutput.ContentLength) @@ -242,6 +247,7 @@ func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n if len(p) == 0 { break } + stalled.Reset(s3ReadTimeout) readCount, err = getObjectOutput.Body.Read(p) n += readCount @@ -253,6 +259,9 @@ func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n if err == io.EOF && n == requested { err = nil } + if err != nil && ctx.Err() != nil { + err = fmt.Errorf("bucket %s read %s: no data for %v: %w", s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, s3ReadTimeout, err) + } return } diff --git a/weed/storage/backend/s3_backend/s3_backend_test.go b/weed/storage/backend/s3_backend/s3_backend_test.go index 347c379820..f103469ad0 100644 --- a/weed/storage/backend/s3_backend/s3_backend_test.go +++ b/weed/storage/backend/s3_backend/s3_backend_test.go @@ -7,6 +7,8 @@ import ( "testing" "testing/iotest" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" @@ -23,7 +25,7 @@ type stubS3Client struct { getObject func(*s3.GetObjectInput) (*s3.GetObjectOutput, error) } -func (s *stubS3Client) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { +func (s *stubS3Client) GetObjectWithContext(_ aws.Context, input *s3.GetObjectInput, _ ...request.Option) (*s3.GetObjectOutput, error) { return s.getObject(input) } diff --git a/weed/storage/backend/s3_backend/s3_sessions.go b/weed/storage/backend/s3_backend/s3_sessions.go index bdadb56118..bf9f3af5f0 100644 --- a/weed/storage/backend/s3_backend/s3_sessions.go +++ b/weed/storage/backend/s3_backend/s3_sessions.go @@ -3,6 +3,7 @@ package s3_backend import ( "crypto/tls" "fmt" + "net" "net/http" "sync" "time" @@ -21,8 +22,14 @@ var ( sessionsLock sync.Mutex ) -// httpOptions tunes the HTTP transport of the tier backend's S3 client. The -// zero value keeps the SDK's default client. +// A tier endpoint that drops packets or accepts connections without answering +// must not hold a read, or a volume server's startup, for minutes per request. +var ( + s3ConnectTimeout = 10 * time.Second + s3ReadTimeout = time.Minute +) + +// httpOptions tunes the HTTP transport of the tier backend's S3 client. type httpOptions struct { // disableHTTP2 keeps the client on HTTP/1.1. Over HTTP/2 every upload and // range read to the endpoint shares one connection and its flow-control @@ -36,12 +43,10 @@ type httpOptions struct { idleConnTimeout time.Duration } -// newTransport returns the transport for the options, or nil for the default. func (o httpOptions) newTransport() *http.Transport { - if o == (httpOptions{}) { - return nil - } t := http.DefaultTransport.(*http.Transport).Clone() + t.DialContext = (&net.Dialer{Timeout: s3ConnectTimeout, KeepAlive: 30 * time.Second}).DialContext + t.ResponseHeaderTimeout = s3ReadTimeout if o.disableHTTP2 { t.ForceAttemptHTTP2 = false t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{} @@ -73,9 +78,7 @@ func createSession(awsAccessKeyId, awsSecretAccessKey, region, endpoint string, Endpoint: aws.String(endpoint), S3ForcePathStyle: aws.Bool(forcePathStyle), S3DisableContentMD5Validation: aws.Bool(true), - } - if t := httpOpts.newTransport(); t != nil { - config.HTTPClient = &http.Client{Transport: t} + HTTPClient: &http.Client{Transport: httpOpts.newTransport()}, } if awsAccessKeyId != "" && awsSecretAccessKey != "" { config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, awsSecretAccessKey, "") diff --git a/weed/storage/backend/s3_backend/s3_sessions_test.go b/weed/storage/backend/s3_backend/s3_sessions_test.go index eb9f621c4d..eb5699259c 100644 --- a/weed/storage/backend/s3_backend/s3_sessions_test.go +++ b/weed/storage/backend/s3_backend/s3_sessions_test.go @@ -1,13 +1,18 @@ package s3_backend import ( + "bufio" "math" + "net" "net/http" "strconv" + "strings" + "sync" "testing" "time" "github.com/aws/aws-sdk-go/service/s3" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" ) func backendTransport(t *testing.T, s *S3BackendStorage) *http.Transport { @@ -54,17 +59,103 @@ func TestS3BackendHTTPOptionsReachTheClient(t *testing.T) { } } -func TestS3BackendDefaultHTTPClientUnchanged(t *testing.T) { +func TestS3BackendDefaultTransportHasTimeouts(t *testing.T) { s, err := newS3BackendStorage(testProperties{"endpoint": "http://127.0.0.1:2"}, "", "http-defaults") if err != nil { t.Fatal(err) } - if transport := backendTransport(t, s); transport != nil && transport != http.DefaultTransport { - t.Fatalf("no options set, yet the client got a custom transport %+v", transport) + transport := backendTransport(t, s) + if transport == nil || transport == http.DefaultTransport { + t.Fatalf("the client must not use the shared default transport, got %+v", transport) } - if (httpOptions{}).newTransport() != nil { - t.Fatal("zero options must keep the default client") + if transport.ResponseHeaderTimeout != s3ReadTimeout { + t.Errorf("ResponseHeaderTimeout = %v, want %v", transport.ResponseHeaderTimeout, s3ReadTimeout) } + if transport.DialContext == nil { + t.Error("no dialer with a connect timeout") + } + if !transport.ForceAttemptHTTP2 { + t.Error("default options must keep HTTP/2") + } +} + +// stallingEndpoint accepts connections and holds them open. With a non-empty +// head, it answers each request with head and then sends nothing more. +func stallingEndpoint(t *testing.T, head string) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + var conns []net.Conn + var mu sync.Mutex + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + mu.Lock() + conns = append(conns, c) + mu.Unlock() + if head != "" { + go func() { + if _, err := http.ReadRequest(bufio.NewReader(c)); err == nil { + c.Write([]byte(head)) + } + }() + } + } + }() + t.Cleanup(func() { + ln.Close() + mu.Lock() + defer mu.Unlock() + for _, c := range conns { + c.Close() + } + }) + return "http://" + ln.Addr().String() +} + +func readFromStallingEndpoint(t *testing.T, head, wantErr string) { + old := s3ReadTimeout + s3ReadTimeout = 100 * time.Millisecond + defer func() { s3ReadTimeout = old }() + + s, err := newS3BackendStorage(testProperties{ + "endpoint": stallingEndpoint(t, head), + "region": "us-east-1", + "bucket": "b", + "aws_access_key_id": "k", + "aws_secret_access_key": "s", + }, "", "stalling") + if err != nil { + t.Fatal(err) + } + f := s.NewStorageFile("key", &volume_server_pb.VolumeInfo{Files: []*volume_server_pb.RemoteFile{{FileSize: 1024}}}) + + done := make(chan error, 1) + go func() { + _, err := f.ReadAt(make([]byte, 8), 0) + done <- err + }() + select { + case err := <-done: + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("want an error containing %q from a stalled endpoint, got %v", wantErr, err) + } + case <-time.After(30 * time.Second): + t.Fatal("a read from a stalled endpoint did not give up") + } +} + +func TestS3BackendReadFailsAgainstSilentEndpoint(t *testing.T) { + readFromStallingEndpoint(t, "", "timeout") +} + +func TestS3BackendReadFailsWhenBodyStalls(t *testing.T) { + readFromStallingEndpoint(t, "HTTP/1.1 206 Partial Content\r\nContent-Length: 8\r\nContent-Range: bytes 0-7/1024\r\n\r\n", "no data for") } func TestS3BackendSessionsDifferByHTTPOptions(t *testing.T) { diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go index 0707acef14..f0b07e21db 100644 --- a/weed/storage/needle/needle_parse_upload.go +++ b/weed/storage/needle/needle_parse_upload.go @@ -96,7 +96,7 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p } } else if r.FormValue("type") != "replicate" { // replica writes must keep the source needle's compression state, not re-derive it - ext := filepath.Base(pu.FileName) + ext := filepath.Ext(pu.FileName) mimeType := pu.MimeType if mimeType == "" { mimeType = http.DetectContentType(pu.Data) @@ -298,7 +298,9 @@ func parseUpload(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) { pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm")) - if !pu.IsChunkedFile { + if r.FormValue("type") == "replicate" { + pu.MimeType = contentType + } else if !pu.IsChunkedFile { dotIndex := strings.LastIndex(pu.FileName, ".") ext, mtype := "", "" diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go index ae2fd07c1b..6e464c0845 100644 --- a/weed/storage/volume_loading.go +++ b/weed/storage/volume_loading.go @@ -232,6 +232,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind return datFileLoadError(v.FileName(".dat"), err) } + var remoteSuperBlockErr error if alreadyHasSuperBlock { err = v.readSuperBlock() if err == nil { @@ -244,6 +245,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind if v.HasRemoteFile() { // maybe temporary network problem logRemoteSuperBlockErr(v.Id, err) + remoteSuperBlockErr = err err = nil } } else { @@ -292,7 +294,9 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind // Recover rows that deletes on a tiered read-only volume overwrote at // the front of .idx. Best effort: a volume that cannot be repaired is // still servable for everything the surviving rows index. - if restored, repairErr := v.repairIdxHeadTombstones(); repairErr != nil { + if remoteSuperBlockErr != nil { + glog.V(1).Infof("volume %d: remote backend unavailable, not recovering overwritten %s rows on this load", v.Id, v.FileName(".idx")) + } else if restored, repairErr := v.repairIdxHeadTombstones(); repairErr != nil { glog.Warningf("volume %d: recover overwritten %s rows: %v", v.Id, v.FileName(".idx"), repairErr) } else if restored > 0 { glog.V(0).Infof("volume %d: recovered %d overwritten %s rows from %s", v.Id, restored, v.FileName(".idx"), v.FileName(".dat")) diff --git a/weed/storage/volume_remote_superblock_stall_test.go b/weed/storage/volume_remote_superblock_stall_test.go new file mode 100644 index 0000000000..4aa43a7c35 --- /dev/null +++ b/weed/storage/volume_remote_superblock_stall_test.go @@ -0,0 +1,185 @@ +package storage + +import ( + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/backend" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// stallingBackend serves localDirBackend's objects, but every read waits +// until release is closed: an endpoint that accepts requests and never answers. +type stallingBackend struct { + *localDirBackend + release chan struct{} + reads atomic.Int32 +} + +func (b *stallingBackend) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) backend.BackendStorageFile { + return &stallingBackendFile{BackendStorageFile: b.localDirBackend.NewStorageFile(key, tierInfo), backend: b} +} + +type stallingBackendFile struct { + backend.BackendStorageFile + backend *stallingBackend +} + +func (f *stallingBackendFile) ReadAt(p []byte, off int64) (int, error) { + f.backend.reads.Add(1) + <-f.backend.release + return f.BackendStorageFile.ReadAt(p, off) +} + +func shortenRemoteSuperBlockWait(t *testing.T, timeout, stall time.Duration) { + t.Helper() + oldTimeout, oldStall := remoteSuperBlockTimeout, remoteSuperBlockStallPeriod + remoteSuperBlockTimeout, remoteSuperBlockStallPeriod = timeout, stall + remoteSuperBlockStalls.Delete(testBackendName) + t.Cleanup(func() { + remoteSuperBlockTimeout, remoteSuperBlockStallPeriod = oldTimeout, oldStall + remoteSuperBlockStalls.Delete(testBackendName) + }) +} + +func TestReadRemoteSuperBlockStallSkipsBackend(t *testing.T) { + shortenRemoteSuperBlockWait(t, 50*time.Millisecond, time.Hour) + b := &stallingBackend{localDirBackend: newLocalDirBackend(t), release: make(chan struct{})} + key, _, err := b.CopyFile(writeSuperBlockFile(t), nil, 0) + require.NoError(t, err) + f := b.NewStorageFile(key, &volume_server_pb.VolumeInfo{Files: []*volume_server_pb.RemoteFile{{FileSize: super_block.SuperBlockSize}}}) + + start := time.Now() + _, err = readRemoteSuperBlock(testBackendName, f) + require.Error(t, err) + require.Less(t, time.Since(start), 5*time.Second) + require.EqualValues(t, 1, b.reads.Load()) + + _, err = readRemoteSuperBlock(testBackendName, f) + require.Error(t, err) + require.EqualValues(t, 1, b.reads.Load(), "a stalled backend must not be asked again within the stall period") + + close(b.release) + remoteSuperBlockStalls.Store(testBackendName, time.Now().Add(-time.Second)) + sb, err := readRemoteSuperBlock(testBackendName, f) + require.NoError(t, err) + require.Equal(t, needle.GetCurrentVersion(), sb.Version) + _, stalled := remoteSuperBlockStalls.Load(testBackendName) + require.False(t, stalled, "a successful read clears the stall") +} + +func writeSuperBlockFile(t *testing.T) *os.File { + t.Helper() + f, err := os.Create(filepath.Join(t.TempDir(), "sb.dat")) + require.NoError(t, err) + t.Cleanup(func() { f.Close() }) + sb := super_block.SuperBlock{Version: needle.GetCurrentVersion(), ReplicaPlacement: &super_block.ReplicaPlacement{}, Ttl: &needle.TTL{}} + _, err = f.Write(sb.Bytes()) + require.NoError(t, err) + return f +} + +// A backend that never answers must not hold up the disk scan, local volumes +// included, and the tiered volumes must serve reads once it answers again. +func TestRemoteTier_DiskScanDoesNotWaitOnStalledBackend(t *testing.T) { + shortenRemoteSuperBlockWait(t, 200*time.Millisecond, time.Hour) + local := newLocalDirBackend(t) + registerTestBackend(t, local) + + dir := t.TempDir() + remoteVids := []needle.VolumeId{61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72} + for _, vid := range remoteVids { + tierUpVolume(t, dir, vid, local) + } + lv, err := NewVolume(dir, dir, "", 60, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0) + require.NoError(t, err) + lv.Close() + + stalling := &stallingBackend{localDirBackend: local, release: make(chan struct{})} + backend.BackendStorages[testBackendName] = stalling + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(stalling.release) }) } + t.Cleanup(release) + + loc := &DiskLocation{ + Directory: dir, + DirectoryUuid: "test-uuid", + IdxDirectory: dir, + DiskType: types.HddType, + MaxVolumeCount: 100, + OriginalMaxVolumeCount: 100, + MinFreeSpace: util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"}, + } + loc.volumes = make(map[needle.VolumeId]*Volume) + loc.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume) + + start := time.Now() + loc.concurrentLoadingVolumes(NeedleMapInMemory, 2, 0, 0) + require.Less(t, time.Since(start), 5*time.Second, "the disk scan waited on the stalled backend") + defer func() { + for _, v := range loc.volumes { + v.Close() + } + }() + + _, ok := loc.FindVolume(60) + require.True(t, ok, "local volume not loaded") + for _, vid := range remoteVids { + v, ok := loc.FindVolume(vid) + require.True(t, ok, "tiered volume %d not loaded", vid) + require.True(t, v.HasRemoteFile()) + } + require.Less(t, int(stalling.reads.Load()), len(remoteVids), "every tiered volume waited on the stalled backend") + + release() + v, _ := loc.FindVolume(remoteVids[0]) + _, err = v.readNeedle(newEmptyNeedle(1), nil, nil) + require.NoError(t, err, "a tiered volume must serve reads once the backend answers") +} + +// The .idx head repair reads the remote .dat, so it must wait for a load that +// can reach the backend instead of stalling this one. +func TestRemoteTier_IdxHeadRepairWaitsForReachableBackend(t *testing.T) { + shortenRemoteSuperBlockWait(t, 200*time.Millisecond, time.Hour) + local := newLocalDirBackend(t) + registerTestBackend(t, local) + + dir := t.TempDir() + const vid = needle.VolumeId(80) + tierUpVolume(t, dir, vid, local) + idxPath := filepath.Join(dir, "80.idx") + clobberIdxHead(t, idxPath, []uint64{4, 5}) + sizeBefore := fileSize(t, idxPath) + + stalling := &stallingBackend{localDirBackend: local, release: make(chan struct{})} + backend.BackendStorages[testBackendName] = stalling + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(stalling.release) }) } + time.AfterFunc(10*time.Second, release) + t.Cleanup(release) + + start := time.Now() + v := reloadVolume(t, dir, vid) + require.Less(t, time.Since(start), 5*time.Second, "the load waited on the stalled backend") + v.Close() + require.Equal(t, sizeBefore, fileSize(t, idxPath), "the repair ran against an unreachable backend") + + backend.BackendStorages[testBackendName] = local + remoteSuperBlockStalls.Delete(testBackendName) + v = reloadVolume(t, dir, vid) + defer v.Close() + require.Greater(t, fileSize(t, idxPath), sizeBefore, "the repair did not run once the backend answered") + _, err := v.readNeedle(newEmptyNeedle(1), nil, nil) + require.NoError(t, err) +} diff --git a/weed/storage/volume_super_block.go b/weed/storage/volume_super_block.go index 7035d9867c..b6faf0ea07 100644 --- a/weed/storage/volume_super_block.go +++ b/weed/storage/volume_super_block.go @@ -3,6 +3,8 @@ package storage import ( "fmt" "os" + "sync" + "time" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/storage/backend" @@ -41,7 +43,11 @@ func (v *Volume) maybeWriteSuperBlock(ver needle.Version) error { } func (v *Volume) readSuperBlock() (err error) { - v.SuperBlock, err = super_block.ReadSuperBlock(v.DataBackend) + if v.HasRemoteFile() && len(v.volumeInfo.GetFiles()) > 0 { + v.SuperBlock, err = readRemoteSuperBlock(v.volumeInfo.GetFiles()[0].BackendName(), v.DataBackend) + } else { + v.SuperBlock, err = super_block.ReadSuperBlock(v.DataBackend) + } if v.volumeInfo != nil && v.volumeInfo.Replication != "" { if replication, err := super_block.NewReplicaPlacementFromString(v.volumeInfo.Replication); err != nil { return fmt.Errorf("Error parse volume %d replication %s : %v", v.Id, v.volumeInfo.Replication, err) @@ -51,3 +57,38 @@ func (v *Volume) readSuperBlock() (err error) { } return err } + +// A tiered volume loads without its superblock when the backend cannot serve +// it; once a read stalls, the backend's other volumes skip it for a while. +var ( + remoteSuperBlockTimeout = 10 * time.Second + remoteSuperBlockStallPeriod = time.Minute + remoteSuperBlockStalls sync.Map +) + +func readRemoteSuperBlock(backendName string, datBackend backend.BackendStorageFile) (super_block.SuperBlock, error) { + if until, ok := remoteSuperBlockStalls.Load(backendName); ok && time.Now().Before(until.(time.Time)) { + return super_block.SuperBlock{}, fmt.Errorf("backend %s stalled on a super block read, skipped", backendName) + } + type result struct { + superBlock super_block.SuperBlock + err error + } + done := make(chan result, 1) + go func() { + sb, err := super_block.ReadSuperBlock(datBackend) + done <- result{sb, err} + }() + timer := time.NewTimer(remoteSuperBlockTimeout) + defer timer.Stop() + select { + case r := <-done: + if r.err == nil { + remoteSuperBlockStalls.Delete(backendName) + } + return r.superBlock, r.err + case <-timer.C: + remoteSuperBlockStalls.Store(backendName, time.Now().Add(remoteSuperBlockStallPeriod)) + return super_block.SuperBlock{}, fmt.Errorf("backend %s did not return the super block within %v", backendName, remoteSuperBlockTimeout) + } +} diff --git a/weed/util/compression.go b/weed/util/compression.go index d62ba90883..aac6b1e012 100644 --- a/weed/util/compression.go +++ b/weed/util/compression.go @@ -99,6 +99,12 @@ func IsZstdContent(data []byte) bool { /* * Default not to compressed since compression can be done on client side. */func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool) { + ext = strings.ToLower(ext) + // Match on the media type alone: "application/json; charset=utf-8" is JSON. + if i := strings.IndexByte(mtype, ';'); i >= 0 { + mtype = strings.TrimSpace(mtype[:i]) + } + mtype = strings.ToLower(mtype) // text if strings.HasPrefix(mtype, "text/") { @@ -137,6 +143,10 @@ func IsZstdContent(data []byte) bool { if strings.HasSuffix(mtype, "script") { return true, true } + // application/json, application/ld+json, application/x-ndjson, application/json-seq, ... + if strings.HasSuffix(mtype, "json") || strings.HasSuffix(mtype, "json-seq") { + return true, true + } if strings.HasSuffix(mtype, "vnd.rar") { return false, true } diff --git a/weed/util/compression_test.go b/weed/util/compression_test.go new file mode 100644 index 0000000000..e9486be3e1 --- /dev/null +++ b/weed/util/compression_test.go @@ -0,0 +1,31 @@ +package util + +import "testing" + +func TestIsCompressableFileType(t *testing.T) { + tests := []struct { + ext, mtype string + compress, isSure bool + }{ + {".json", "", true, true}, + {".JSON", "", true, true}, + {"", "application/json", true, true}, + {"", "application/json; charset=utf-8", true, true}, + {"", "application/ld+json", true, true}, + {"", "application/x-ndjson", true, true}, + {"", "application/json-seq", true, true}, + {"", "Application/JSON", true, true}, + {"", "text/plain; charset=utf-8", true, true}, + {"", "application/xml", true, true}, + {".gz", "application/json", false, true}, + {"", "application/gzip", false, false}, + {"", "image/png", false, true}, + {"", "", false, false}, + } + for _, tt := range tests { + compress, isSure := IsCompressableFileType(tt.ext, tt.mtype) + if compress != tt.compress || isSure != tt.isSure { + t.Errorf("IsCompressableFileType(%q, %q) = (%v, %v), want (%v, %v)", tt.ext, tt.mtype, compress, isSure, tt.compress, tt.isSure) + } + } +} diff --git a/weed/util/fullpath.go b/weed/util/fullpath.go index 866592f95d..529344a548 100644 --- a/weed/util/fullpath.go +++ b/weed/util/fullpath.go @@ -76,6 +76,15 @@ func (fp FullPath) IsLongerFileName(maxFilenameLength uint32) bool { return uint32(len([]byte(fp.Name()))) > maxFilenameLength } +func (fp FullPath) HasDotSegment() bool { + for _, seg := range strings.Split(string(fp), "/") { + if seg == "." || seg == ".." { + return true + } + } + return false +} + func (fp FullPath) Child(name string) FullPath { dir := string(fp) noPrefix := name diff --git a/weed/util/version/constants.go b/weed/util/version/constants.go index 2e3c4c5468..0cf26abbe7 100644 --- a/weed/util/version/constants.go +++ b/weed/util/version/constants.go @@ -9,7 +9,7 @@ import ( var ( MAJOR_VERSION = int32(4) - MINOR_VERSION = int32(51) + MINOR_VERSION = int32(52) VERSION_NUMBER = fmt.Sprintf("%d.%02d", MAJOR_VERSION, MINOR_VERSION) VERSION = util.SizeLimit + " " + VERSION_NUMBER COMMIT = ""