mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-13 15:29:12 +01:00
Merge branch 'master' into fix-delete-offline-replica
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
64f92bfa8a
155 files changed
+12108
-2073
No files matched your search
@@ -0,0 +1,37 @@
|
||||
name: "Cancel runs of closed PRs"
|
||||
|
||||
# Merging or closing a PR leaves its pull_request runs going: concurrency
|
||||
# groups cancel a run only when a newer push arrives. pull_request_target gives
|
||||
# this job a token that can cancel runs, fork PRs included. It checks out and
|
||||
# runs no PR code.
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [closed]
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
cancel:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Cancel the PR's queued and running pull_request runs
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
PR: ${{ github.event.pull_request.number }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.ref }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Match the PR's own commits: forks reuse branch names such as master.
|
||||
shas=$(gh api --paginate "repos/$GH_REPO/pulls/$PR/commits" --jq '.[].sha')
|
||||
for status in queued in_progress waiting pending requested; do
|
||||
gh run list --branch "$HEAD_REF" --event pull_request --status "$status" \
|
||||
--limit 200 --json databaseId,headSha --jq '.[] | "\(.databaseId) \(.headSha)"'
|
||||
done | while read -r id sha; do
|
||||
grep -qx "$sha" <<<"$shas" || continue
|
||||
echo "cancel run $id at $sha"
|
||||
gh run cancel "$id" || true
|
||||
done
|
||||
@@ -8,6 +8,7 @@ on:
|
||||
- 'seaweed-volume/**'
|
||||
- 'seaweed-common/**'
|
||||
- 'seaweed-worker/**'
|
||||
- 'seaweed-vector/**'
|
||||
- 'docker/**'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
@@ -50,7 +51,8 @@ jobs:
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
seaweed-worker/target/${{ matrix.target }}/release
|
||||
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
|
||||
seaweed-vector/target/${{ matrix.target }}/release
|
||||
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-dev-${{ matrix.target }}-
|
||||
|
||||
@@ -83,6 +85,12 @@ jobs:
|
||||
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
|
||||
|
||||
- name: Build the vector engine
|
||||
run: |
|
||||
cd seaweed-vector
|
||||
cargo build --release -p weed-vector-server --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
@@ -90,6 +98,7 @@ jobs:
|
||||
path: |
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
weed-worker-${{ matrix.arch }}
|
||||
weed-vector-${{ matrix.arch }}
|
||||
|
||||
build-dev-containers:
|
||||
needs: [build-rust-binaries]
|
||||
@@ -127,8 +136,17 @@ jobs:
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-vector-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-vector-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
|
||||
echo "Placed pre-built vector engine for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
ls -la docker/weed-vector-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -96,7 +96,8 @@ jobs:
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
seaweed-worker/target/${{ matrix.target }}/release
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
|
||||
seaweed-vector/target/${{ matrix.target }}/release
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
@@ -137,6 +138,17 @@ jobs:
|
||||
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
|
||||
|
||||
- name: Build the vector engine
|
||||
run: |
|
||||
# An older source_ref predates the engine; its image goes without one.
|
||||
if [ ! -f seaweed-vector/crates/server/Cargo.toml ]; then
|
||||
echo "No vector engine in this source ref"
|
||||
exit 0
|
||||
fi
|
||||
cd seaweed-vector
|
||||
cargo build --release -p weed-vector-server --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
@@ -145,6 +157,7 @@ jobs:
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
weed-worker-${{ matrix.arch }}
|
||||
weed-vector-${{ matrix.arch }}
|
||||
|
||||
build:
|
||||
needs: [setup, build-rust-binaries]
|
||||
@@ -217,8 +230,17 @@ jobs:
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-vector-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-vector-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
|
||||
echo "Placed pre-built vector engine for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
ls -la docker/weed-vector-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
@@ -342,8 +364,17 @@ jobs:
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-vector-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-vector-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
|
||||
echo "Placed pre-built vector engine for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
ls -la docker/weed-vector-prebuilt/
|
||||
- name: Create BuildKit config for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
|
||||
@@ -43,9 +43,9 @@ concurrency:
|
||||
jobs:
|
||||
|
||||
# ── Pre-build the Rust binaries natively ────────────────────────────
|
||||
# The volume server and the Rust maintenance worker, cross-compiled for
|
||||
# amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into
|
||||
# ~15 minutes of native compilation.
|
||||
# The volume server, the Rust maintenance worker and the vector engine,
|
||||
# cross-compiled for amd64 and arm64 without QEMU, turning a 5-hour emulated
|
||||
# cargo build into ~15 minutes of native compilation.
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
@@ -73,7 +73,8 @@ jobs:
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
seaweed-worker/target/${{ matrix.target }}/release
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
|
||||
seaweed-vector/target/${{ matrix.target }}/release
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
@@ -114,6 +115,12 @@ jobs:
|
||||
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
|
||||
|
||||
- name: Build the vector engine
|
||||
run: |
|
||||
cd seaweed-vector
|
||||
cargo build --release -p weed-vector-server --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
@@ -122,6 +129,7 @@ jobs:
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
weed-worker-${{ matrix.arch }}
|
||||
weed-vector-${{ matrix.arch }}
|
||||
|
||||
# One job per (variant, platform) on a native runner, pushed by digest;
|
||||
# the merge job stitches the digests into one multi-arch tag.
|
||||
@@ -198,8 +206,17 @@ jobs:
|
||||
echo "Placed pre-built Rust worker for ${arch}"
|
||||
fi
|
||||
done
|
||||
mkdir -p docker/weed-vector-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-vector-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
|
||||
echo "Placed pre-built vector engine for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
ls -la docker/weed-worker-prebuilt/
|
||||
ls -la docker/weed-vector-prebuilt/
|
||||
|
||||
- name: Free Disk Space
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
|
||||
@@ -68,19 +68,6 @@ jobs:
|
||||
restore-keys: |
|
||||
rust-vector-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
|
||||
# lance's build scripts compile their own protos and look for a protoc.
|
||||
# Point them at the one protoc-bin-vendored ships, so no job depends on a
|
||||
# system package and every build sees the same version.
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-vector
|
||||
cargo fetch
|
||||
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
|
||||
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
|
||||
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build the vector crates
|
||||
run: cd seaweed-vector && cargo build --release
|
||||
|
||||
@@ -125,15 +112,6 @@ jobs:
|
||||
key: vectors-e2e-${{ runner.os }}-${{ runner.arch }}-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-vector/Cargo.lock', 'seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
vectors-e2e-${{ runner.os }}-${{ runner.arch }}-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-vector
|
||||
cargo fetch
|
||||
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
|
||||
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
|
||||
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
- name: Test Go vector gateway
|
||||
run: |
|
||||
go test ./weed/s3api/s3vectors -count=1
|
||||
|
||||
@@ -188,6 +188,83 @@ jobs:
|
||||
weed-worker_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
# The vector engine behind the S3 Vectors API: Linux only, like the worker,
|
||||
# whose lance dependency tree it shares.
|
||||
build-rust-vector-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
asset_suffix: linux_amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
asset_suffix: linux_arm64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up the musl target
|
||||
uses: ./.github/actions/rust-musl-target
|
||||
with:
|
||||
target: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
if: runner.environment == 'github-hosted'
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-vector/target/${{ matrix.target }}/release
|
||||
key: rust-vector-release-${{ matrix.target }}-${{ hashFiles('seaweed-vector/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-vector-release-${{ matrix.target }}-
|
||||
|
||||
- name: Use the vendored protoc
|
||||
run: |
|
||||
cd seaweed-vector
|
||||
cargo fetch
|
||||
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
|
||||
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
|
||||
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
|
||||
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
|
||||
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build the vector engine
|
||||
run: |
|
||||
cd seaweed-vector
|
||||
cargo build --release -p weed-vector-server --target ${{ matrix.target }}
|
||||
|
||||
- name: Package binary
|
||||
run: |
|
||||
cp seaweed-vector/target/${{ matrix.target }}/release/weed-vector weed-vector
|
||||
tar czf weed-vector_${{ matrix.asset_suffix }}.tar.gz weed-vector
|
||||
rm weed-vector
|
||||
md5sum weed-vector_${{ matrix.asset_suffix }}.tar.gz > weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-vector_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-vector-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-vector_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5
|
||||
|
||||
build-rust-volume-darwin:
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
@@ -33,6 +33,7 @@ ARG TARGETARCH
|
||||
ARG TAGS
|
||||
COPY weed-volume-prebuilt/ /prebuilt/
|
||||
COPY weed-worker-prebuilt/ /prebuilt-worker/
|
||||
COPY weed-vector-prebuilt/ /prebuilt-vector/
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
|
||||
# seaweed-common is a path dependency of seaweed-volume that lives beside it,
|
||||
# so the source build below needs it in the same relative position.
|
||||
@@ -65,11 +66,19 @@ RUN if [ -f "/prebuilt-worker/weed-worker-${TARGETARCH}" ]; then \
|
||||
echo "No pre-built Rust worker for ${TARGETARCH}" && \
|
||||
touch /weed-worker; \
|
||||
fi
|
||||
# The vector engine carries the same lance tree, so it is pre-built or absent too.
|
||||
RUN if [ -f "/prebuilt-vector/weed-vector-${TARGETARCH}" ]; then \
|
||||
echo "Using pre-built vector engine for ${TARGETARCH}" && \
|
||||
cp "/prebuilt-vector/weed-vector-${TARGETARCH}" /weed-vector; \
|
||||
else \
|
||||
echo "No pre-built vector engine for ${TARGETARCH}" && \
|
||||
touch /weed-vector; \
|
||||
fi
|
||||
|
||||
# Pre-built binaries arrive via GitHub Actions artifacts, which drop the
|
||||
# executable bit, so the copied file is 0644 and exec fails with "Permission
|
||||
# denied". Restore it (no-op for the empty placeholders, which stay size 0).
|
||||
RUN chmod 0755 /weed-volume /weed-worker
|
||||
RUN chmod 0755 /weed-volume /weed-worker /weed-vector
|
||||
|
||||
FROM alpine AS final
|
||||
LABEL author="Chris Lu"
|
||||
@@ -78,6 +87,8 @@ COPY --from=builder /go/bin/weed /usr/bin/
|
||||
COPY --from=rust_builder /weed-volume /usr/bin/weed-volume
|
||||
# Same for the Rust maintenance worker, which serves Lance table buckets
|
||||
COPY --from=rust_builder /weed-worker /usr/bin/weed-worker
|
||||
# And the vector engine behind the S3 Vectors API
|
||||
COPY --from=rust_builder /weed-vector /usr/bin/weed-vector
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
@@ -100,6 +100,19 @@ case "$1" in
|
||||
exec /usr/bin/weed-worker "$@"
|
||||
;;
|
||||
|
||||
'vector')
|
||||
shift
|
||||
if [ ! -s /usr/bin/weed-vector ]; then
|
||||
echo "Error: the vector engine is not available on this platform ($(uname -m))." >&2
|
||||
exit 1
|
||||
fi
|
||||
# The gateway dials the engine from another container, so not loopback.
|
||||
if [ -z "$WEED_VECTOR_LISTEN" ] && ! isArgPassed "-listen" "$@"; then
|
||||
set -- --listen 0.0.0.0:8090 "$@"
|
||||
fi
|
||||
exec /usr/bin/weed-vector "$@"
|
||||
;;
|
||||
|
||||
'server')
|
||||
ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024"
|
||||
if isArgPassed "-volume.max" "$@"; then
|
||||
|
||||
Whitespace-only changes.
+52
-2
@@ -9,7 +9,7 @@
|
||||
# curl -fsSL ... | bash -s -- --version 4.34 --dir /usr/local/bin
|
||||
#
|
||||
# Options:
|
||||
# --component COMP Which binary to install: weed, volume-rust, worker-rust, all (default: weed)
|
||||
# --component COMP Which binary to install: weed, volume-rust, worker-rust, vector, all (default: weed)
|
||||
# --version VER Release version tag (default: latest)
|
||||
# --large-disk Use large disk variant (5-byte offset, 8TB max volume)
|
||||
# --dir DIR Installation directory (default: /usr/local/bin)
|
||||
@@ -23,6 +23,7 @@ VERSION=""
|
||||
LARGE_DISK=false
|
||||
INSTALL_DIR="/usr/local/bin"
|
||||
WORKER_INSTALLED=false
|
||||
VECTOR_INSTALLED=false
|
||||
|
||||
# Colors (if terminal supports them)
|
||||
if [ -t 1 ]; then
|
||||
@@ -156,6 +157,12 @@ worker_asset_name() {
|
||||
echo "weed-worker_${os}_${arch}.tar.gz"
|
||||
}
|
||||
|
||||
# The vector engine behind the S3 Vectors API, built like the worker.
|
||||
vector_asset_name() {
|
||||
local os="$1" arch="$2"
|
||||
echo "weed-vector_${os}_${arch}.tar.gz"
|
||||
}
|
||||
|
||||
# Install a single component
|
||||
install_component() {
|
||||
local component="$1" os="$2" arch="$3"
|
||||
@@ -264,8 +271,38 @@ install_component() {
|
||||
ok "Installed weed-worker to ${INSTALL_DIR}/weed-worker"
|
||||
;;
|
||||
|
||||
vector)
|
||||
# Published for linux amd64/arm64 only, like the worker.
|
||||
case "$os" in
|
||||
linux) ;;
|
||||
*) error "The vector engine is not available for ${os}. Supported: linux" ;;
|
||||
esac
|
||||
case "$arch" in
|
||||
amd64|arm64) ;;
|
||||
*) error "The vector engine is not available for ${arch}. Supported: amd64, arm64" ;;
|
||||
esac
|
||||
|
||||
asset_name="$(vector_asset_name "$os" "$arch")"
|
||||
download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
|
||||
download "$download_url" "${tmpdir}/${asset_name}"
|
||||
|
||||
info "Extracting ${asset_name}..."
|
||||
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
|
||||
|
||||
local vector_bin
|
||||
vector_bin="$(find "$tmpdir" -name 'weed-vector' -type f | head -1)"
|
||||
if [ -z "$vector_bin" ]; then
|
||||
error "Could not find weed-vector binary in archive"
|
||||
fi
|
||||
|
||||
chmod +x "$vector_bin"
|
||||
install_binary "$vector_bin" "weed-vector"
|
||||
VECTOR_INSTALLED=true
|
||||
ok "Installed weed-vector to ${INSTALL_DIR}/weed-vector"
|
||||
;;
|
||||
|
||||
*)
|
||||
error "Unknown component: ${component}. Use: weed, volume-rust, worker-rust, all"
|
||||
error "Unknown component: ${component}. Use: weed, volume-rust, worker-rust, vector, all"
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -327,6 +364,13 @@ main() {
|
||||
else
|
||||
install_component "worker-rust" "$os" "$arch"
|
||||
fi
|
||||
if [ "$os" != "linux" ] || { [ "$arch" != "amd64" ] && [ "$arch" != "arm64" ]; }; then
|
||||
warn "Skipping the vector engine: no build for ${os}/${arch}"
|
||||
elif ! asset_exists "https://github.com/${REPO}/releases/download/${VERSION}/$(vector_asset_name "$os" "$arch")"; then
|
||||
warn "Skipping the vector engine: ${VERSION} does not carry one"
|
||||
else
|
||||
install_component "vector" "$os" "$arch"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
install_component "$COMPONENT" "$os" "$arch"
|
||||
@@ -344,6 +388,9 @@ main() {
|
||||
if [ "$WORKER_INSTALLED" = true ]; then
|
||||
info " weed-worker: ${INSTALL_DIR}/weed-worker"
|
||||
fi
|
||||
if [ "$VECTOR_INSTALLED" = true ]; then
|
||||
info " weed-vector: ${INSTALL_DIR}/weed-vector"
|
||||
fi
|
||||
echo ""
|
||||
info "Quick start:"
|
||||
info " weed master # Start master server"
|
||||
@@ -352,6 +399,9 @@ main() {
|
||||
if [ "$WORKER_INSTALLED" = true ]; then
|
||||
info " weed-worker --admin localhost:12535 # Start the Rust maintenance worker"
|
||||
fi
|
||||
if [ "$VECTOR_INSTALLED" = true ]; then
|
||||
info " weed-vector --help # The vector engine for S3 vector buckets"
|
||||
fi
|
||||
}
|
||||
|
||||
main
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.51"
|
||||
appVersion: "4.52"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.51.0
|
||||
version: 4.52.0
|
||||
+1008
-1005
File diff suppressed because it is too large.
Load diff
|
Before Width: | Height: | Size: 53 KiB After Width: | Height: | Size: 53 KiB |
@@ -0,0 +1,6 @@
|
||||
# lance's build scripts run $PROTOC. Point them at the protoc that
|
||||
# protoc-bin-vendored ships, as this workspace's own build script does, so a
|
||||
# build needs no system protoc. An explicit PROTOC still wins. Cargo reads this
|
||||
# file only when it runs inside seaweed-vector/.
|
||||
[env]
|
||||
PROTOC = { value = ".cargo/protoc", relative = true }
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
# Runs the protoc of the protoc-bin-vendored version in Cargo.lock, or the one
|
||||
# on PATH when the registry does not hold it.
|
||||
case "$(uname -s)-$(uname -m)" in
|
||||
Linux-x86_64) platform=linux-x86_64 ;;
|
||||
Linux-aarch64) platform=linux-aarch_64 ;;
|
||||
Linux-ppc64le) platform=linux-ppcle_64 ;;
|
||||
Linux-s390x) platform=linux-s390_64 ;;
|
||||
Linux-i?86) platform=linux-x86_32 ;;
|
||||
Darwin-x86_64) platform=macos-x86_64 ;;
|
||||
Darwin-arm64) platform=macos-aarch_64 ;;
|
||||
*) exec protoc "$@" ;;
|
||||
esac
|
||||
crate="protoc-bin-vendored-$platform"
|
||||
version=$(awk -v name="\"$crate\"" '$1 == "name" && $3 == name { found = 1; next } found && $1 == "version" { gsub(/"/, "", $3); print $3; exit }' "$(dirname "$0")/../Cargo.lock")
|
||||
for protoc in "${CARGO_HOME:-$HOME/.cargo}"/registry/src/*/"$crate-$version"/bin/protoc; do
|
||||
if [ -x "$protoc" ]; then
|
||||
exec "$protoc" "$@"
|
||||
fi
|
||||
done
|
||||
exec protoc "$@"
|
||||
@@ -1,14 +1,16 @@
|
||||
//! Lance's json_extract and json_exists parse their JSONPath again for every
|
||||
//! row. A filter's path is a literal, so swap in copies that parse it once per
|
||||
//! batch and return the same values.
|
||||
//! batch and return the same values. json_array_contains is swapped too, and
|
||||
//! compares the elements as typed JSON values: Lance's compares their text, so
|
||||
//! 1 would match "1".
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::builder::{BooleanBuilder, StringBuilder};
|
||||
use arrow_array::{Array, ArrayRef, LargeBinaryArray, new_null_array};
|
||||
use arrow_schema::DataType;
|
||||
use jsonb::RawJsonb;
|
||||
use jsonb::jsonpath::{Selector, parse_json_path};
|
||||
use jsonb::{OwnedJsonb, RawJsonb};
|
||||
use lance::dataset::scanner::Scanner;
|
||||
use lance::deps::datafusion::common::tree_node::{Transformed, TreeNode};
|
||||
use lance::deps::datafusion::common::{DataFusionError, Result, ScalarValue};
|
||||
@@ -34,12 +36,16 @@ fn swap(expr: Expr) -> Result<Expr> {
|
||||
let op = match f.func.name() {
|
||||
"json_extract" => Op::Extract,
|
||||
"json_exists" => Op::Exists,
|
||||
"json_array_contains" => Op::Contains,
|
||||
_ => return Ok(Transformed::no(Expr::ScalarFunction(f))),
|
||||
};
|
||||
if !matches!(
|
||||
f.args.get(1),
|
||||
Some(Expr::Literal(ScalarValue::Utf8(Some(_)), _))
|
||||
) {
|
||||
let literal = |i: usize| {
|
||||
matches!(
|
||||
f.args.get(i),
|
||||
Some(Expr::Literal(ScalarValue::Utf8(Some(_)), _))
|
||||
)
|
||||
};
|
||||
if !literal(1) || (op == Op::Contains && !literal(2)) {
|
||||
return Ok(Transformed::no(Expr::ScalarFunction(f)));
|
||||
}
|
||||
let udf = Arc::new(ScalarUDF::new_from_impl(JsonPath::new(op)));
|
||||
@@ -54,6 +60,7 @@ fn swap(expr: Expr) -> Result<Expr> {
|
||||
enum Op {
|
||||
Extract,
|
||||
Exists,
|
||||
Contains,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Hash)]
|
||||
@@ -64,12 +71,13 @@ struct JsonPath {
|
||||
|
||||
impl JsonPath {
|
||||
fn new(op: Op) -> Self {
|
||||
let mut args = vec![DataType::LargeBinary, DataType::Utf8];
|
||||
if op == Op::Contains {
|
||||
args.push(DataType::Utf8);
|
||||
}
|
||||
Self {
|
||||
op,
|
||||
signature: Signature::exact(
|
||||
vec![DataType::LargeBinary, DataType::Utf8],
|
||||
Volatility::Immutable,
|
||||
),
|
||||
signature: Signature::exact(args, Volatility::Immutable),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -83,6 +91,7 @@ impl ScalarUDFImpl for JsonPath {
|
||||
match self.op {
|
||||
Op::Extract => "json_extract",
|
||||
Op::Exists => "json_exists",
|
||||
Op::Contains => "json_array_contains",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,17 +102,24 @@ impl ScalarUDFImpl for JsonPath {
|
||||
fn return_type(&self, _: &[DataType]) -> Result<DataType> {
|
||||
Ok(match self.op {
|
||||
Op::Extract => DataType::Utf8,
|
||||
Op::Exists => DataType::Boolean,
|
||||
Op::Exists | Op::Contains => DataType::Boolean,
|
||||
})
|
||||
}
|
||||
|
||||
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
|
||||
let [data, ColumnarValue::Scalar(ScalarValue::Utf8(Some(path)))] = args.args.as_slice()
|
||||
else {
|
||||
return Err(exec_err(format!(
|
||||
"{} expects a literal JSONPath",
|
||||
self.name()
|
||||
)));
|
||||
let (data, path, value) = match args.args.as_slice() {
|
||||
[data, ColumnarValue::Scalar(ScalarValue::Utf8(Some(path)))] => (data, path, None),
|
||||
[
|
||||
data,
|
||||
ColumnarValue::Scalar(ScalarValue::Utf8(Some(path))),
|
||||
ColumnarValue::Scalar(ScalarValue::Utf8(Some(value))),
|
||||
] => (data, path, Some(value)),
|
||||
_ => {
|
||||
return Err(exec_err(format!(
|
||||
"{} expects literal arguments",
|
||||
self.name()
|
||||
)));
|
||||
}
|
||||
};
|
||||
let data = data.to_array(args.number_rows)?;
|
||||
let data = data
|
||||
@@ -160,6 +176,37 @@ impl ScalarUDFImpl for JsonPath {
|
||||
}
|
||||
Arc::new(builder.finish())
|
||||
}
|
||||
Op::Contains => {
|
||||
let value = value.map(String::as_str).unwrap_or_default();
|
||||
let value: OwnedJsonb = value
|
||||
.parse()
|
||||
.map_err(|e| exec_err(format!("Invalid JSON value '{value}': {e}")))?;
|
||||
let mut builder = BooleanBuilder::with_capacity(data.len());
|
||||
for i in 0..data.len() {
|
||||
if data.is_null(i) {
|
||||
builder.append_null();
|
||||
continue;
|
||||
}
|
||||
let mut selector = Selector::new(RawJsonb::new(data.value(i)));
|
||||
let found = match selector.select_value(&json_path) {
|
||||
Ok(Some(array)) => array
|
||||
.as_raw()
|
||||
.array_values()
|
||||
.map_err(|e| {
|
||||
exec_err(format!("Failed to read array at '{path}': {e}"))
|
||||
})?
|
||||
.is_some_and(|items| items.contains(&value)),
|
||||
Ok(None) => false,
|
||||
Err(e) => {
|
||||
return Err(exec_err(format!(
|
||||
"Failed to check array contains at path '{path}': {e}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
builder.append_value(found);
|
||||
}
|
||||
Arc::new(builder.finish())
|
||||
}
|
||||
};
|
||||
Ok(ColumnarValue::Array(out))
|
||||
}
|
||||
@@ -298,6 +345,48 @@ mod tests {
|
||||
assert!(keys(&dataset, bad, true).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn array_contains_compares_typed_values() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let metas = [
|
||||
Some(r#"{"k": ["space", "classic"]}"#),
|
||||
Some(r#"{"k": [1, 2.5]}"#),
|
||||
Some(r#"{"k": ["1", "2.5"]}"#),
|
||||
Some(r#"{"k": [true]}"#),
|
||||
Some(r#"{"k": ["true"]}"#),
|
||||
Some(r#"{"k": "space"}"#),
|
||||
Some(r#"{"k": [["space"]]}"#),
|
||||
Some(r#"{"k": {"a": "space"}}"#),
|
||||
Some(r#"{"k": ["", "spa\"ce"]}"#),
|
||||
Some(r#"{"k": []}"#),
|
||||
Some(r#"{"x": 1}"#),
|
||||
None,
|
||||
];
|
||||
let dataset = dataset(&dir, &metas).await;
|
||||
for (value, want) in [
|
||||
(r#""space""#, vec!["k0"]),
|
||||
("1", vec!["k1"]),
|
||||
("1.0", vec!["k1"]),
|
||||
("2.5", vec!["k1"]),
|
||||
(r#""1""#, vec!["k2"]),
|
||||
("true", vec!["k3"]),
|
||||
(r#""true""#, vec!["k4"]),
|
||||
(r#""""#, vec!["k8"]),
|
||||
(r#""spa\"ce""#, vec!["k8"]),
|
||||
(r#""classic""#, vec!["k0"]),
|
||||
("3", vec![]),
|
||||
] {
|
||||
let filter = format!(r#"json_array_contains(`metadata`, '$["k"]', '{value}')"#);
|
||||
assert_eq!(
|
||||
keys(&dataset, &filter, true).await.unwrap(),
|
||||
want,
|
||||
"{filter}"
|
||||
);
|
||||
}
|
||||
let bad = r#"json_array_contains(`metadata`, '$["k"]', 'not json')"#;
|
||||
assert!(keys(&dataset, bad, true).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn apply_swaps_literal_paths_only() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
@@ -305,7 +394,7 @@ mod tests {
|
||||
let mut scanner = dataset.scan();
|
||||
apply(
|
||||
&mut scanner,
|
||||
r#"json_extract(`metadata`, '$["k"]') = '"v"' AND json_exists(`metadata`, '$["k"]') AND json_get_string(`metadata`, 'k') = 'v'"#,
|
||||
r#"json_extract(`metadata`, '$["k"]') = '"v"' AND json_exists(`metadata`, '$["k"]') AND json_get_string(`metadata`, 'k') = 'v' AND json_array_contains(`metadata`, '$["k"]', '"v"') AND NOT json_array_contains(`metadata`, '$["k"]', key)"#,
|
||||
)
|
||||
.unwrap();
|
||||
let mut swapped = Vec::new();
|
||||
@@ -325,6 +414,8 @@ mod tests {
|
||||
assert_eq!(
|
||||
swapped,
|
||||
[
|
||||
("json_array_contains".to_string(), false),
|
||||
("json_array_contains".to_string(), true),
|
||||
("json_exists".to_string(), true),
|
||||
("json_extract".to_string(), true),
|
||||
("json_get_string".to_string(), false),
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
use anyhow::Result;
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Parser;
|
||||
use tonic::transport::Server;
|
||||
use weed_lance_worker::catalog::NamespaceClient;
|
||||
use tonic::transport::server::TcpIncoming;
|
||||
use weed_lance_worker::catalog::{NamespaceClient, SigningKey};
|
||||
use weed_lance_worker::dataset::FallbackOptions;
|
||||
use weed_vector_server::pb::vector::vector_service_server::VectorServiceServer;
|
||||
use weed_vector_server::read_cache::ReadCache;
|
||||
@@ -10,15 +11,15 @@ use weed_vector_server::service::VectorServer;
|
||||
/// The VectorService the S3 gateway fans vector calls out to.
|
||||
#[derive(Parser)]
|
||||
struct Args {
|
||||
/// Lance Namespace base URL, e.g. http://localhost:8888/lance.
|
||||
/// Lance Namespace base URL: the S3 gateway's Lance port (-s3.port.lance).
|
||||
#[arg(
|
||||
long,
|
||||
env = "WEED_VECTOR_NAMESPACE",
|
||||
default_value = "http://localhost:8888/lance"
|
||||
default_value = "http://localhost:9101"
|
||||
)]
|
||||
namespace: String,
|
||||
/// gRPC listen address.
|
||||
#[arg(long, env = "WEED_VECTOR_LISTEN", default_value = "127.0.0.1:8080")]
|
||||
#[arg(long, env = "WEED_VECTOR_LISTEN", default_value = "127.0.0.1:8090")]
|
||||
listen: String,
|
||||
/// object_store option fallback for deployments that vend no credentials:
|
||||
/// key=value, repeatable.
|
||||
@@ -31,6 +32,10 @@ struct Args {
|
||||
/// Memory for caching reads of write-once table files, in MiB; 0 disables.
|
||||
#[arg(long, env = "WEED_VECTOR_READ_CACHE_MB", default_value_t = 512)]
|
||||
read_cache_mb: u64,
|
||||
/// The `jwt.filer_signing.key` of the cluster. The Lance namespace vends
|
||||
/// credentials that write index data only to a caller holding it.
|
||||
#[arg(long, env = "WEED_FILER_SIGNING_KEY")]
|
||||
filer_signing_key: Option<String>,
|
||||
}
|
||||
|
||||
fn parse_option(s: &str) -> Result<(String, String), String> {
|
||||
@@ -100,6 +105,11 @@ fn advertised_addr(listen: &str, masters: &[String]) -> String {
|
||||
async fn main() -> Result<()> {
|
||||
tracing_subscriber::fmt::init();
|
||||
let args = Args::parse_from(normalize_args(std::env::args().collect()));
|
||||
// Bind before registering, so the master never lists an engine that is
|
||||
// about to exit because its port is taken.
|
||||
let incoming = TcpIncoming::bind(args.listen.parse()?)
|
||||
.with_context(|| format!("listen on {}", args.listen))?
|
||||
.with_nodelay(Some(true));
|
||||
if let Some(masters) = &args.masters {
|
||||
let masters: Vec<String> = masters.split(',').map(|s| s.trim().to_string()).collect();
|
||||
for master in &masters {
|
||||
@@ -111,7 +121,10 @@ async fn main() -> Result<()> {
|
||||
));
|
||||
}
|
||||
let fallback: FallbackOptions = args.storage_options.into_iter().collect();
|
||||
let client = NamespaceClient::new(args.namespace.clone()).with_fallback(&fallback);
|
||||
let signing_key = SigningKey::new(args.filer_signing_key);
|
||||
let client = NamespaceClient::new(args.namespace.clone())
|
||||
.with_fallback(&fallback)
|
||||
.with_signing_key(signing_key.as_ref());
|
||||
let mut service = VectorServer::new(client, fallback);
|
||||
if args.read_cache_mb > 0 {
|
||||
service = service.with_read_cache(ReadCache::new(args.read_cache_mb << 20));
|
||||
@@ -125,7 +138,7 @@ async fn main() -> Result<()> {
|
||||
.max_decoding_message_size(1 << 30)
|
||||
.max_encoding_message_size(1 << 30),
|
||||
)
|
||||
.serve(args.listen.parse()?)
|
||||
.serve_with_incoming(incoming)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -424,13 +424,13 @@ impl VectorService for VectorServer {
|
||||
if dimension == 0 || dimension > 4096 {
|
||||
return Err(Status::invalid_argument("dimension must be 1..=4096"));
|
||||
}
|
||||
// The gateway has already created the table through the catalog.
|
||||
let description = self
|
||||
.client
|
||||
.declare_table(&table.id, None)
|
||||
.describe_table(&table.id)
|
||||
.await
|
||||
.map_err(internal)?;
|
||||
let mut options = self.fallback.clone();
|
||||
options.extend(description.storage_options.clone());
|
||||
let options = dataset::table_options(&description, &self.fallback);
|
||||
let schema = schema_for(dimension, metric, &req.non_filterable_metadata_keys);
|
||||
let empty = RecordBatch::new_empty(schema.clone());
|
||||
let reader = arrow_array::RecordBatchIterator::new(vec![Ok(empty)].into_iter(), schema);
|
||||
|
||||
@@ -143,12 +143,12 @@ fn flip_horizontal_if(img: DynamicImage, do_flip: bool) -> DynamicImage {
|
||||
DynamicImage::ImageRgba8(out)
|
||||
}
|
||||
|
||||
/// Returns true if the given MIME type or file path extension indicates a JPEG file.
|
||||
pub fn is_jpeg(mime_type: &str, path: &str) -> bool {
|
||||
/// Returns true if the given MIME type or file name extension indicates a JPEG file.
|
||||
pub fn is_jpeg(mime_type: &str, filename: &str) -> bool {
|
||||
if mime_type == "image/jpeg" {
|
||||
return true;
|
||||
}
|
||||
let lower = path.to_lowercase();
|
||||
let lower = filename.to_lowercase();
|
||||
lower.ends_with(".jpg") || lower.ends_with(".jpeg")
|
||||
}
|
||||
|
||||
|
||||
@@ -5,21 +5,153 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, OnceLock, RwLock};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::config::http::HttpResponse;
|
||||
use aws_sdk_s3::config::timeout::TimeoutConfig;
|
||||
use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
|
||||
use aws_sdk_s3::error::{DisplayErrorContext, SdkError};
|
||||
use aws_sdk_s3::operation::get_object::GetObjectError;
|
||||
use aws_sdk_s3::operation::head_object::HeadObjectError;
|
||||
use aws_sdk_s3::operation::upload_part::UploadPartOutput;
|
||||
use aws_sdk_s3::operation::upload_part::builders::UploadPartFluentBuilder;
|
||||
use aws_sdk_s3::primitives::{ByteStream, SdkBody};
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use bytes::Bytes;
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio::sync::{Semaphore, watch};
|
||||
|
||||
/// Concurrency limit for multipart upload/download (matches Go's s3manager).
|
||||
const CONCURRENCY: usize = 5;
|
||||
|
||||
/// A tier endpoint that drops packets or accepts connections without answering
|
||||
/// must not hold a read, or a volume server's startup, for minutes per request.
|
||||
/// Matches Go's s3ConnectTimeout and s3ReadTimeout. The SDK's stalled-stream
|
||||
/// protection already fails a body that stops arriving.
|
||||
const S3_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const S3_READ_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
/// The SDK's read timeout runs from the start of the request, so on
|
||||
/// upload_part it would also cap sending the part body. send_part bounds the
|
||||
/// response wait instead, from when the body is written, like Go's
|
||||
/// ResponseHeaderTimeout.
|
||||
fn upload_part_config() -> aws_sdk_s3::config::Builder {
|
||||
aws_sdk_s3::config::Builder::default().timeout_config(
|
||||
TimeoutConfig::builder()
|
||||
.connect_timeout(S3_CONNECT_TIMEOUT)
|
||||
.disable_read_timeout()
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
/// A part goes to the connection in chunks this size, so its last chunk is
|
||||
/// taken only once nearly all of the part has been written.
|
||||
const UPLOAD_CHUNK_SIZE: usize = 64 * 1024;
|
||||
|
||||
/// The latest attempt to send a part, and whether its body has been sent.
|
||||
type PartSent = Arc<watch::Sender<(u64, bool)>>;
|
||||
|
||||
struct PartBody {
|
||||
rest: Bytes,
|
||||
attempt: u64,
|
||||
sent: PartSent,
|
||||
}
|
||||
|
||||
impl http_body::Body for PartBody {
|
||||
type Data = Bytes;
|
||||
type Error = std::convert::Infallible;
|
||||
|
||||
fn poll_frame(
|
||||
mut self: Pin<&mut Self>,
|
||||
_cx: &mut Context<'_>,
|
||||
) -> Poll<Option<Result<http_body::Frame<Bytes>, Self::Error>>> {
|
||||
if self.rest.is_empty() {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
let n = self.rest.len().min(UPLOAD_CHUNK_SIZE);
|
||||
let chunk = self.rest.split_to(n);
|
||||
if self.rest.is_empty() {
|
||||
let attempt = self.attempt;
|
||||
self.sent.send_if_modified(|state| {
|
||||
let latest = state.0 == attempt;
|
||||
state.1 |= latest;
|
||||
latest
|
||||
});
|
||||
}
|
||||
Poll::Ready(Some(Ok(http_body::Frame::data(chunk))))
|
||||
}
|
||||
|
||||
fn is_end_stream(&self) -> bool {
|
||||
self.rest.is_empty()
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> http_body::SizeHint {
|
||||
http_body::SizeHint::with_exact(self.rest.len() as u64)
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends one multipart part, giving up once `response_timeout` passes after
|
||||
/// the whole part was handed to the connection without a response. A retry
|
||||
/// rebuilds the body and restarts that clock. The body keeps its in-memory
|
||||
/// contents, so signing and the checksum header are unchanged.
|
||||
async fn send_part(
|
||||
request: UploadPartFluentBuilder,
|
||||
data: Bytes,
|
||||
response_timeout: Duration,
|
||||
) -> Result<UploadPartOutput, String> {
|
||||
let sent: PartSent = Arc::new(watch::channel((0, false)).0);
|
||||
let body = {
|
||||
let sent = sent.clone();
|
||||
SdkBody::from(data.clone()).map_preserve_contents(move |_| {
|
||||
let mut attempt = 0;
|
||||
sent.send_modify(|state| {
|
||||
*state = (state.0 + 1, data.is_empty());
|
||||
attempt = state.0;
|
||||
});
|
||||
SdkBody::from_body_1_x(PartBody {
|
||||
rest: data.clone(),
|
||||
attempt,
|
||||
sent: sent.clone(),
|
||||
})
|
||||
})
|
||||
};
|
||||
tokio::select! {
|
||||
resp = request
|
||||
.body(ByteStream::new(body))
|
||||
.customize()
|
||||
.config_override(upload_part_config())
|
||||
.send() => resp.map_err(|e| DisplayErrorContext(&e).to_string()),
|
||||
() = wait_unanswered(sent, response_timeout) => Err(format!(
|
||||
"no response within {response_timeout:?} after sending the part"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Completes once an attempt's body has been sent and `response_timeout`
|
||||
/// passes before a later attempt starts.
|
||||
async fn wait_unanswered(sent: PartSent, response_timeout: Duration) {
|
||||
let mut state = sent.subscribe();
|
||||
loop {
|
||||
let attempt = state
|
||||
.wait_for(|&(_, done)| done)
|
||||
.await
|
||||
.map_or(0, |state| state.0);
|
||||
if tokio::time::timeout(
|
||||
response_timeout,
|
||||
state.wait_for(|&(latest, _)| latest != attempt),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A tier transfer failure. The variant is what callers match on; the
|
||||
/// message is the operator-facing text.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
@@ -107,7 +239,13 @@ impl S3TierBackend {
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.region(Region::new(region.to_string()))
|
||||
.credentials_provider(credentials)
|
||||
.force_path_style(config.force_path_style);
|
||||
.force_path_style(config.force_path_style)
|
||||
.timeout_config(
|
||||
TimeoutConfig::builder()
|
||||
.connect_timeout(S3_CONNECT_TIMEOUT)
|
||||
.read_timeout(S3_READ_TIMEOUT)
|
||||
.build(),
|
||||
);
|
||||
|
||||
if !config.endpoint.is_empty() {
|
||||
s3_config = s3_config.endpoint_url(&config.endpoint);
|
||||
@@ -231,23 +369,23 @@ impl S3TierBackend {
|
||||
TierError::Io(format!("failed to read file at offset {}: {}", off, e))
|
||||
})?;
|
||||
|
||||
let upload_part_resp = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(pn)
|
||||
.body(buf.into())
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
TierError::Io(format!(
|
||||
"failed to upload part {} at offset {}: {}",
|
||||
pn,
|
||||
off,
|
||||
DisplayErrorContext(&e)
|
||||
))
|
||||
})?;
|
||||
let upload_part_resp = send_part(
|
||||
client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(pn),
|
||||
buf.into(),
|
||||
S3_READ_TIMEOUT,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
TierError::Io(format!(
|
||||
"failed to upload part {} at offset {}: {}",
|
||||
pn, off, e
|
||||
))
|
||||
})?;
|
||||
|
||||
let e_tag = upload_part_resp.e_tag().unwrap_or_default().to_string();
|
||||
|
||||
@@ -560,6 +698,25 @@ impl S3TierBackend {
|
||||
})
|
||||
}
|
||||
|
||||
/// `read_range_blocking` that gives up once `timeout` passes, returning
|
||||
/// `Ok(None)`.
|
||||
pub fn read_range_blocking_within(
|
||||
self: &Arc<Self>,
|
||||
key: &str,
|
||||
offset: u64,
|
||||
size: usize,
|
||||
timeout: Duration,
|
||||
) -> Result<Option<Vec<u8>>, TierError> {
|
||||
let backend = Arc::clone(self);
|
||||
let key = key.to_string();
|
||||
block_on_tier_future(async move {
|
||||
match tokio::time::timeout(timeout, backend.read_range(&key, offset, size)).await {
|
||||
Ok(read) => read.map(Some),
|
||||
Err(_) => Ok(None),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn read_range_blocking(
|
||||
&self,
|
||||
key: &str,
|
||||
@@ -725,6 +882,13 @@ fn describe_join_error(join_error: tokio::task::JoinError) -> TierError {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::remote_storage::s3::tests::{CannedResponse, NO_SUCH_KEY};
|
||||
use aws_sdk_s3::config::http::HttpRequest;
|
||||
use aws_sdk_s3::config::{HttpClient, RuntimeComponents};
|
||||
use aws_smithy_runtime_api::client::http::{
|
||||
HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector,
|
||||
};
|
||||
use aws_smithy_runtime_api::http::StatusCode;
|
||||
use http_body::Body as _;
|
||||
use std::collections::HashSet;
|
||||
use tokio::runtime::Handle;
|
||||
|
||||
@@ -897,6 +1061,235 @@ mod tests {
|
||||
assert!(matches!(err, TierError::Io(_)), "{err:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backend_bounds_connect_and_read_waits() {
|
||||
let (_listener, backend) = silent_backend();
|
||||
let timeouts = backend.client.config().timeout_config().unwrap();
|
||||
assert_eq!(timeouts.connect_timeout(), Some(S3_CONNECT_TIMEOUT));
|
||||
assert_eq!(timeouts.read_timeout(), Some(S3_READ_TIMEOUT));
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct CaptureTimeouts(Arc<std::sync::Mutex<Option<TimeoutConfig>>>);
|
||||
|
||||
impl aws_sdk_s3::config::Intercept for CaptureTimeouts {
|
||||
fn name(&self) -> &'static str {
|
||||
"CaptureTimeouts"
|
||||
}
|
||||
|
||||
fn read_before_transmit(
|
||||
&self,
|
||||
_context: &aws_sdk_s3::config::interceptors::BeforeTransmitInterceptorContextRef<'_>,
|
||||
_runtime_components: &aws_sdk_s3::config::RuntimeComponents,
|
||||
cfg: &mut aws_sdk_s3::config::ConfigBag,
|
||||
) -> Result<(), aws_sdk_s3::error::BoxError> {
|
||||
*self.0.lock().unwrap() = cfg.load::<TimeoutConfig>().cloned();
|
||||
Err("captured".into())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upload_part_keeps_the_connect_timeout_without_the_read_timeout() {
|
||||
let (_listener, backend) = silent_backend();
|
||||
let seen = Arc::new(std::sync::Mutex::new(None));
|
||||
let _ = backend
|
||||
.client
|
||||
.upload_part()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.upload_id("upload")
|
||||
.part_number(1)
|
||||
.body(vec![0u8; 8].into())
|
||||
.customize()
|
||||
.config_override(upload_part_config().interceptor(CaptureTimeouts(seen.clone())))
|
||||
.send()
|
||||
.await;
|
||||
let timeouts = seen
|
||||
.lock()
|
||||
.unwrap()
|
||||
.clone()
|
||||
.expect("request never reached transmit");
|
||||
assert_eq!(timeouts.connect_timeout(), Some(S3_CONNECT_TIMEOUT));
|
||||
assert_eq!(timeouts.read_timeout(), None);
|
||||
}
|
||||
|
||||
fn upload_part_request(backend: &S3TierBackend) -> UploadPartFluentBuilder {
|
||||
backend
|
||||
.client
|
||||
.upload_part()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.upload_id("upload")
|
||||
.part_number(1)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn send_part_gives_up_when_no_response_follows_the_body() {
|
||||
let (_listener, backend) = silent_backend();
|
||||
let sent = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
send_part(
|
||||
upload_part_request(&backend),
|
||||
Bytes::from_static(&[0u8; 8]),
|
||||
Duration::from_millis(200),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("a part with no response did not give up");
|
||||
let err = sent.unwrap_err();
|
||||
assert!(err.starts_with("no response within 200ms"), "{err}");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_retry_restarts_the_response_clock() {
|
||||
let sent: PartSent = Arc::new(watch::channel((1, true)).0);
|
||||
let mut unanswered =
|
||||
std::pin::pin!(wait_unanswered(sent.clone(), Duration::from_millis(200)));
|
||||
let wait = |ms| tokio::time::sleep(Duration::from_millis(ms));
|
||||
tokio::select! {
|
||||
() = unanswered.as_mut() => panic!("gave up early"),
|
||||
() = wait(150) => {}
|
||||
}
|
||||
// The retry starts and finishes sending before the waiter runs again.
|
||||
sent.send_replace((2, false));
|
||||
sent.send_replace((2, true));
|
||||
tokio::select! {
|
||||
() = unanswered.as_mut() => panic!("kept the first attempt's clock"),
|
||||
() = wait(150) => {}
|
||||
}
|
||||
tokio::select! {
|
||||
() = unanswered => {}
|
||||
() = wait(100) => panic!("the retry never timed out"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Takes the request body one chunk per `pause`, then answers at once.
|
||||
#[derive(Debug, Clone)]
|
||||
struct SlowReader {
|
||||
pause: Duration,
|
||||
}
|
||||
|
||||
impl HttpConnector for SlowReader {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
let pause = self.pause;
|
||||
HttpConnectorFuture::new(async move {
|
||||
let mut body = std::pin::pin!(request.into_body());
|
||||
loop {
|
||||
tokio::time::sleep(pause).await;
|
||||
let frame = std::future::poll_fn(|cx| body.as_mut().poll_frame(cx)).await;
|
||||
if frame.is_none() || body.is_end_stream() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let mut response =
|
||||
HttpResponse::new(StatusCode::try_from(200).unwrap(), SdkBody::empty());
|
||||
response.headers_mut().insert("ETag", "\"etag\"");
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl HttpClient for SlowReader {
|
||||
fn http_connector(
|
||||
&self,
|
||||
_settings: &HttpConnectorSettings,
|
||||
_components: &RuntimeComponents,
|
||||
) -> SharedHttpConnector {
|
||||
SharedHttpConnector::new(self.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn send_part_does_not_cap_a_slow_body() {
|
||||
let config = aws_sdk_s3::Config::builder()
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.region(Region::new("us-east-1"))
|
||||
.credentials_provider(Credentials::new("AKIATEST", "secret", None, None, "test"))
|
||||
.endpoint_url("http://127.0.0.1:1")
|
||||
.force_path_style(true)
|
||||
.http_client(SlowReader {
|
||||
pause: Duration::from_millis(300),
|
||||
})
|
||||
.build();
|
||||
let backend = S3TierBackend {
|
||||
client: Client::from_conf(config),
|
||||
bucket: "bucket".to_string(),
|
||||
storage_class: "STANDARD".to_string(),
|
||||
};
|
||||
let output = send_part(
|
||||
upload_part_request(&backend),
|
||||
Bytes::from(vec![0u8; 4 * UPLOAD_CHUNK_SIZE]),
|
||||
Duration::from_millis(200),
|
||||
)
|
||||
.await
|
||||
.expect("a part still being sent was cut short");
|
||||
assert_eq!(output.e_tag(), Some("\"etag\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_range_blocking_within_gives_up_on_a_silent_endpoint() {
|
||||
let (_listener, backend) = silent_backend();
|
||||
let start = std::time::Instant::now();
|
||||
let got = backend
|
||||
.read_range_blocking_within("key", 0, 8, Duration::from_millis(200))
|
||||
.unwrap();
|
||||
assert!(got.is_none());
|
||||
assert!(start.elapsed() < Duration::from_secs(5));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_range_gives_up_when_the_body_stalls() {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let endpoint = format!("http://{}", listener.local_addr().unwrap());
|
||||
std::thread::spawn(move || {
|
||||
let mut held = Vec::new();
|
||||
for mut conn in listener.incoming().flatten() {
|
||||
let mut request = Vec::new();
|
||||
let mut byte = [0u8; 1];
|
||||
while !request.ends_with(b"\r\n\r\n") {
|
||||
match std::io::Read::read(&mut conn, &mut byte) {
|
||||
Ok(1) => request.push(byte[0]),
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
let _ = std::io::Write::write_all(
|
||||
&mut conn,
|
||||
b"HTTP/1.1 206 Partial Content\r\nContent-Length: 8\r\nContent-Range: bytes 0-7/1024\r\n\r\n",
|
||||
);
|
||||
held.push(conn);
|
||||
}
|
||||
});
|
||||
let backend = S3TierBackend::new(&S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
bucket: "bucket".to_string(),
|
||||
endpoint,
|
||||
storage_class: String::new(),
|
||||
force_path_style: true,
|
||||
});
|
||||
|
||||
let read = tokio::time::timeout(Duration::from_secs(30), backend.read_range("key", 0, 8))
|
||||
.await
|
||||
.expect("a read whose body stalls did not give up");
|
||||
let err = read.unwrap_err();
|
||||
assert!(matches!(err, TierError::Io(_)), "{err:?}");
|
||||
}
|
||||
|
||||
fn silent_backend() -> (std::net::TcpListener, Arc<S3TierBackend>) {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let backend = S3TierBackend::new(&S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
bucket: "bucket".to_string(),
|
||||
endpoint: format!("http://{}", listener.local_addr().unwrap()),
|
||||
storage_class: String::new(),
|
||||
force_path_style: true,
|
||||
});
|
||||
(listener, Arc::new(backend))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_range_blocking_no_such_key_is_not_found() {
|
||||
let err = backend_answering(404, NO_SUCH_KEY)
|
||||
|
||||
@@ -733,6 +733,27 @@ pub(crate) async fn check_delete_replicas(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `query` with the fields named in `overrides` replaced by their values.
|
||||
fn override_query_fields(query: &str, overrides: &[(&str, String)]) -> Result<String, String> {
|
||||
let mut fields = Vec::new();
|
||||
for field in query.split('&').filter(|field| !field.is_empty()) {
|
||||
let decoded: Vec<(String, String)> = serde_urlencoded::from_str(field)
|
||||
.map_err(|e| format!("parse replication query: {e}"))?;
|
||||
if !decoded
|
||||
.iter()
|
||||
.any(|(name, _)| overrides.iter().any(|(o, _)| o == name))
|
||||
{
|
||||
fields.push(field);
|
||||
}
|
||||
}
|
||||
let encoded = serde_urlencoded::to_string(overrides)
|
||||
.map_err(|e| format!("encode replication query: {e}"))?;
|
||||
if !encoded.is_empty() {
|
||||
fields.push(&encoded);
|
||||
}
|
||||
Ok(fields.join("&"))
|
||||
}
|
||||
|
||||
/// Helper to synchronously replicate a request to peer volume servers.
|
||||
async fn do_replicated_request(
|
||||
state: &VolumeServerState,
|
||||
@@ -794,16 +815,7 @@ async fn do_replicated_request(
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut query_fields = Vec::new();
|
||||
for field in query.split('&').filter(|field| !field.is_empty()) {
|
||||
let decoded: Vec<(String, String)> = serde_urlencoded::from_str(field)
|
||||
.map_err(|e| format!("parse replication query: {e}"))?;
|
||||
if !decoded.iter().any(|(name, _)| name == "type") {
|
||||
query_fields.push(field);
|
||||
}
|
||||
}
|
||||
query_fields.push("type=replicate");
|
||||
let new_query = query_fields.join("&");
|
||||
let new_query = override_query_fields(query, &[("type", "replicate".to_string())])?;
|
||||
|
||||
let mut futures = Vec::new();
|
||||
for loc in remote_locations {
|
||||
@@ -831,6 +843,12 @@ async fn do_replicated_request(
|
||||
if let Some(auth) = headers.get(axum::http::header::AUTHORIZATION) {
|
||||
req_builder = req_builder.header(axum::http::header::AUTHORIZATION, auth);
|
||||
}
|
||||
for (name, value) in headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.as_str().starts_with("seaweed-"))
|
||||
{
|
||||
req_builder = req_builder.header(name, value);
|
||||
}
|
||||
|
||||
if let Some(ref b) = body {
|
||||
req_builder = req_builder.body(b.clone());
|
||||
@@ -2893,8 +2911,12 @@ pub async fn post_handler(
|
||||
while let Ok(Some(field)) = multipart.next_field().await {
|
||||
let field_name = field.name().map(|s| s.to_string());
|
||||
let fname = field.file_name().map(clean_windows_path_base);
|
||||
let fct = field.content_type().map(|m| m.to_string());
|
||||
let field_headers = field.headers().clone();
|
||||
let fct = field_headers
|
||||
.get(header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| s.to_string());
|
||||
let fce = field_headers
|
||||
.get(header::CONTENT_ENCODING)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
@@ -3060,15 +3082,18 @@ pub async fn post_handler(
|
||||
let mime_type = if let Some(ref pct) = parsed_content_type {
|
||||
pct.clone()
|
||||
} else {
|
||||
let multipart_fallback =
|
||||
if should_parse_multipart && !filename.is_empty() && !is_chunk_manifest {
|
||||
mime_guess::from_path(&filename)
|
||||
.first()
|
||||
.map(|m| m.to_string())
|
||||
.unwrap_or_default()
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
let multipart_fallback = if should_parse_multipart
|
||||
&& !filename.is_empty()
|
||||
&& !is_chunk_manifest
|
||||
&& !is_replicate
|
||||
{
|
||||
mime_guess::from_path(&filename)
|
||||
.first()
|
||||
.map(|m| m.to_string())
|
||||
.unwrap_or_default()
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
headers
|
||||
.get(header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
@@ -3131,23 +3156,24 @@ pub async fn post_handler(
|
||||
.collect();
|
||||
|
||||
// Fix JPEG orientation from EXIF data before storing (matches Go behavior).
|
||||
let body_data = if state.fix_jpg_orientation && crate::images::is_jpeg(&mime_type, &path) {
|
||||
crate::images::fix_jpg_orientation(&body_data_raw)
|
||||
} else {
|
||||
body_data_raw
|
||||
};
|
||||
let (body_data, body_rewritten) =
|
||||
if state.fix_jpg_orientation && crate::images::is_jpeg(&mime_type, &filename) {
|
||||
let fixed = crate::images::fix_jpg_orientation(&body_data_raw);
|
||||
let rewritten = fixed != body_data_raw;
|
||||
(fixed, rewritten)
|
||||
} else {
|
||||
(body_data_raw, false)
|
||||
};
|
||||
|
||||
// Auto-compress compressible file types (matches Go's IsCompressableFileType).
|
||||
// Only compress if not already gzipped and compression saves >10%.
|
||||
// Go uses filepath.Base(pu.FileName) for extension detection (not the URL path).
|
||||
let (final_data, final_is_gzipped) = if !is_gzipped && !is_chunk_manifest {
|
||||
let ext = {
|
||||
let dot_pos = filename.rfind('.');
|
||||
dot_pos
|
||||
.map(|p| filename[p..].to_lowercase())
|
||||
.unwrap_or_default()
|
||||
};
|
||||
if is_compressible_file_type(&ext, &mime_type) {
|
||||
// A replica keeps the compression state the primary stored.
|
||||
let (final_data, final_is_gzipped) = if !is_gzipped && !is_chunk_manifest && !is_replicate {
|
||||
let ext = filename
|
||||
.rfind('.')
|
||||
.map(|p| &filename[p..])
|
||||
.unwrap_or_default();
|
||||
if is_compressible_file_type(ext, &mime_type) {
|
||||
if let Some(compressed) = try_gzip_data(&body_data) {
|
||||
if compressed.len() * 10 < body_data.len() * 9 {
|
||||
(compressed, true)
|
||||
@@ -3235,23 +3261,55 @@ pub async fn post_handler(
|
||||
};
|
||||
|
||||
// The fan-out runs concurrently with the local needle write, like Go's
|
||||
// ReplicatedWrite: it reads the already-buffered request body, so the two
|
||||
// sides share no mutable state. The ack still waits for both.
|
||||
// ReplicatedWrite: replicas get the stored bytes with their compression
|
||||
// state, so they keep it instead of re-deriving it. The ack waits for both.
|
||||
let replication = if needs_replication {
|
||||
let state_clone = state.clone();
|
||||
let path_clone = path.clone();
|
||||
let query_clone = query.clone();
|
||||
let headers_clone = headers.clone();
|
||||
let body_clone = body.clone();
|
||||
let mut overrides = vec![("ts", n.last_modified.to_string())];
|
||||
if let Some(t) = ttl.filter(|t| !t.is_empty()) {
|
||||
overrides.push(("ttl", t.to_string()));
|
||||
}
|
||||
if is_chunk_manifest {
|
||||
overrides.push(("cm", "true".to_string()));
|
||||
}
|
||||
if fsync {
|
||||
overrides.push(("fsync", "true".to_string()));
|
||||
}
|
||||
let mut replica_headers = HeaderMap::new();
|
||||
if let Some(auth) = headers.get(header::AUTHORIZATION) {
|
||||
replica_headers.insert(header::AUTHORIZATION, auth.clone());
|
||||
}
|
||||
if n.has_pairs() {
|
||||
for (key, value) in &pair_map {
|
||||
if let (Ok(name), Ok(value)) = (
|
||||
header::HeaderName::try_from(format!("seaweed-{key}")),
|
||||
header::HeaderValue::from_str(value),
|
||||
) {
|
||||
replica_headers.insert(name, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
let (content_type, replica_body) = replica_upload_body(
|
||||
&filename,
|
||||
&mime_type,
|
||||
n.is_compressed().then_some("gzip"),
|
||||
original_content_md5.as_deref().filter(|_| !body_rewritten),
|
||||
&n.data,
|
||||
);
|
||||
if let Ok(value) = header::HeaderValue::from_str(&content_type) {
|
||||
replica_headers.insert(header::CONTENT_TYPE, value);
|
||||
}
|
||||
Some(tokio::spawn(async move {
|
||||
do_replicated_request(
|
||||
&state_clone,
|
||||
vid.0,
|
||||
Method::POST,
|
||||
&path_clone,
|
||||
&query_clone,
|
||||
&headers_clone,
|
||||
Some(body_clone),
|
||||
&override_query_fields(&query_clone, &overrides)?,
|
||||
&replica_headers,
|
||||
Some(replica_body),
|
||||
)
|
||||
.await
|
||||
}))
|
||||
@@ -4766,6 +4824,15 @@ fn extract_jwt(headers: &HeaderMap, uri: &axum::http::Uri) -> Option<String> {
|
||||
/// Check if a file type should be compressed based on extension and MIME type.
|
||||
/// Returns true only when we are sure the type is compressible.
|
||||
fn is_compressible_file_type(ext: &str, mtype: &str) -> bool {
|
||||
let ext = ext.to_ascii_lowercase();
|
||||
let ext = ext.as_str();
|
||||
let mtype = mtype
|
||||
.split(';')
|
||||
.next()
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_ascii_lowercase();
|
||||
let mtype = mtype.as_str();
|
||||
// text/*
|
||||
if mtype.starts_with("text/") {
|
||||
return true;
|
||||
@@ -4798,6 +4865,9 @@ fn is_compressible_file_type(ext: &str, mtype: &str) -> bool {
|
||||
if mtype.ends_with("script") {
|
||||
return true;
|
||||
}
|
||||
if mtype.ends_with("json") || mtype.ends_with("json-seq") {
|
||||
return true;
|
||||
}
|
||||
if mtype.ends_with("vnd.rar") {
|
||||
return false;
|
||||
}
|
||||
@@ -4811,6 +4881,41 @@ fn is_compressible_file_type(ext: &str, mtype: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// One-part multipart upload of a stored needle, the form Go's upload_content
|
||||
/// sends: the bytes as stored, labeled with their content encoding.
|
||||
fn replica_upload_body(
|
||||
filename: &str,
|
||||
mime: &str,
|
||||
content_encoding: Option<&str>,
|
||||
content_md5: Option<&str>,
|
||||
data: &[u8],
|
||||
) -> (String, bytes::Bytes) {
|
||||
let boundary = uuid::Uuid::new_v4().simple().to_string();
|
||||
let mut head = format!(
|
||||
"--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{}\"\r\n",
|
||||
filename.replace('\\', "\\\\").replace('"', "\\\"")
|
||||
);
|
||||
if !mime.is_empty() {
|
||||
head.push_str(&format!("Content-Type: {mime}\r\n"));
|
||||
}
|
||||
if let Some(encoding) = content_encoding {
|
||||
head.push_str(&format!("Content-Encoding: {encoding}\r\n"));
|
||||
}
|
||||
if let Some(md5) = content_md5 {
|
||||
head.push_str(&format!("Content-MD5: {md5}\r\n"));
|
||||
}
|
||||
head.push_str("\r\n");
|
||||
let tail = format!("\r\n--{boundary}--\r\n");
|
||||
let mut body = Vec::with_capacity(head.len() + data.len() + tail.len());
|
||||
body.extend_from_slice(head.as_bytes());
|
||||
body.extend_from_slice(data);
|
||||
body.extend_from_slice(tail.as_bytes());
|
||||
(
|
||||
format!("multipart/form-data; boundary={boundary}"),
|
||||
body.into(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Try to gzip data. Returns None on error.
|
||||
fn try_gzip_data(data: &[u8]) -> Option<Vec<u8>> {
|
||||
use flate2::Compression;
|
||||
@@ -5283,6 +5388,18 @@ mod tests {
|
||||
assert!(is_compressible_file_type("", "application/javascript"));
|
||||
assert!(!is_compressible_file_type("", "application/zstd"));
|
||||
assert!(!is_compressible_file_type("", "application/vnd.rar"));
|
||||
assert!(is_compressible_file_type("", "application/json"));
|
||||
assert!(is_compressible_file_type(
|
||||
"",
|
||||
"application/json; charset=utf-8"
|
||||
));
|
||||
assert!(is_compressible_file_type("", "Application/JSON"));
|
||||
assert!(is_compressible_file_type("", "application/ld+json"));
|
||||
assert!(is_compressible_file_type("", "application/x-ndjson"));
|
||||
assert!(is_compressible_file_type("", "application/json-seq"));
|
||||
assert!(is_compressible_file_type("", "text/plain; charset=utf-8"));
|
||||
assert!(is_compressible_file_type(".JSON", ""));
|
||||
assert!(!is_compressible_file_type(".gz", "application/json"));
|
||||
|
||||
// Audio
|
||||
assert!(is_compressible_file_type(".wav", "audio/wav"));
|
||||
@@ -8245,12 +8362,15 @@ mod tests {
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
*state.current_master_url.write().await = listener.local_addr().unwrap().to_string();
|
||||
let master_server = tokio::spawn(async move { axum::serve(listener, master).await.unwrap() });
|
||||
let master_server =
|
||||
tokio::spawn(async move { axum::serve(listener, master).await.unwrap() });
|
||||
let id = 0x6e7a_0c05u64;
|
||||
let fid = format!("1,{id:x}{TEST_COOKIE:08x}");
|
||||
let token = gen_jwt(&key, 60, &fid).unwrap();
|
||||
let query = "%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&%74ype=client&type=old";
|
||||
let expected_query = "%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&type=replicate";
|
||||
let query =
|
||||
"%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&ts=1700000000&%74ype=client&type=old";
|
||||
let expected_query =
|
||||
"%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&ts=1700000000&type=replicate";
|
||||
let payload = b"authenticated replica bytes";
|
||||
for method in [Method::POST, Method::DELETE] {
|
||||
let response = tokio::time::timeout(
|
||||
@@ -8299,6 +8419,264 @@ mod tests {
|
||||
peer_server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_replica_stores_the_primary_needle() {
|
||||
use crate::storage::super_block::ReplicaPlacement;
|
||||
use axum::{Json, Router, routing::get};
|
||||
use tower::ServiceExt;
|
||||
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let mut state = volume_test_state(&tmp);
|
||||
Arc::get_mut(&mut state).unwrap().self_url = "127.0.0.1:8080".to_string();
|
||||
let volume = state
|
||||
.store
|
||||
.read()
|
||||
.unwrap()
|
||||
.find_volume(VolumeId(1))
|
||||
.unwrap()
|
||||
.1;
|
||||
volume
|
||||
.set_replica_placement(ReplicaPlacement::from_string("001").unwrap())
|
||||
.unwrap();
|
||||
let peer_tmp = tempfile::TempDir::new().unwrap();
|
||||
let peer = volume_test_state(&peer_tmp);
|
||||
let peer_volume = peer
|
||||
.store
|
||||
.read()
|
||||
.unwrap()
|
||||
.find_volume(VolumeId(1))
|
||||
.unwrap()
|
||||
.1;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let peer_addr = listener.local_addr().unwrap();
|
||||
let peer_server = tokio::spawn(async move {
|
||||
axum::serve(
|
||||
listener,
|
||||
super::super::volume_server::build_admin_router(peer),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
let master = Router::new().route(
|
||||
"/dir/lookup",
|
||||
get(move || async move {
|
||||
Json(serde_json::json!({"locations": [
|
||||
{"url": "127.0.0.1:8080"},
|
||||
{"url": peer_addr.to_string()}
|
||||
]}))
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
*state.current_master_url.write().await = listener.local_addr().unwrap().to_string();
|
||||
let master_server =
|
||||
tokio::spawn(async move { axum::serve(listener, master).await.unwrap() });
|
||||
|
||||
let payload = "compressible replica text ".repeat(64);
|
||||
let boundary = "replica-boundary";
|
||||
let multipart = format!(
|
||||
"--{boundary}\r\nContent-Disposition: form-data; name=\"ttl\"\r\n\r\n3m\r\n\
|
||||
--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"a \\\"b\\\".txt\"\r\n\
|
||||
Content-Type: text/plain\r\n\r\n{payload}\r\n--{boundary}--\r\n"
|
||||
);
|
||||
let cases = [
|
||||
(
|
||||
0x6e7a_0c06u64,
|
||||
Request::builder()
|
||||
.header(header::CONTENT_TYPE, "text/plain")
|
||||
.header(
|
||||
header::CONTENT_DISPOSITION,
|
||||
"attachment; filename=\"notes.txt\"",
|
||||
)
|
||||
.header("Content-MD5", compute_md5_base64(payload.as_bytes()))
|
||||
.header("Seaweed-Color", "blue"),
|
||||
Body::from(payload.clone()),
|
||||
),
|
||||
(
|
||||
0x6e7a_0c07u64,
|
||||
Request::builder().header(
|
||||
header::CONTENT_TYPE,
|
||||
format!("multipart/form-data; boundary={boundary}"),
|
||||
),
|
||||
Body::from(multipart),
|
||||
),
|
||||
(
|
||||
0x6e7a_0c0au64,
|
||||
Request::builder().header(
|
||||
header::CONTENT_DISPOSITION,
|
||||
"attachment; filename=\"untyped.txt\"",
|
||||
),
|
||||
Body::from(payload.clone()),
|
||||
),
|
||||
(
|
||||
0x6e7a_0c0bu64,
|
||||
Request::builder().header(
|
||||
header::CONTENT_DISPOSITION,
|
||||
format!("attachment; filename=\"{}.txt\"", "n".repeat(300)),
|
||||
),
|
||||
Body::from(payload.clone()),
|
||||
),
|
||||
(
|
||||
0x6e7a_0c0eu64,
|
||||
Request::builder()
|
||||
.header(header::CONTENT_TYPE, "Text/Plain; charset=UTF-8")
|
||||
.header(
|
||||
header::CONTENT_DISPOSITION,
|
||||
"attachment; filename=\"upper.txt\"",
|
||||
),
|
||||
Body::from(payload.clone()),
|
||||
),
|
||||
];
|
||||
for (id, request, body) in cases {
|
||||
let response = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
super::super::volume_server::build_admin_router(state.clone()).oneshot(
|
||||
request
|
||||
.method(Method::POST)
|
||||
.uri(format!("/1,{id:x}{TEST_COOKIE:08x}?ts=1700000000"))
|
||||
.body(body)
|
||||
.unwrap(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::CREATED);
|
||||
let read = |v: &Arc<crate::storage::volume::Volume>| {
|
||||
let mut stored = Needle {
|
||||
id: NeedleId(id),
|
||||
cookie: Cookie(TEST_COOKIE),
|
||||
..Needle::default()
|
||||
};
|
||||
v.read_needle(&mut stored).unwrap();
|
||||
stored
|
||||
};
|
||||
let (primary, replica) = (read(&volume), read(&peer_volume));
|
||||
assert!(primary.is_compressed());
|
||||
assert_eq!(
|
||||
maybe_decompress_gzip(&primary.data).unwrap(),
|
||||
payload.as_bytes()
|
||||
);
|
||||
assert_eq!(replica.data, primary.data);
|
||||
assert_eq!(replica.flags, primary.flags);
|
||||
assert_eq!(replica.name, primary.name);
|
||||
assert_eq!(replica.mime, primary.mime);
|
||||
assert_eq!(replica.pairs, primary.pairs);
|
||||
assert_eq!(replica.ttl, primary.ttl);
|
||||
assert_eq!(replica.last_modified, 1700000000);
|
||||
}
|
||||
master_server.abort();
|
||||
peer_server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_replica_keeps_uncompressed_bytes() {
|
||||
use tower::ServiceExt;
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let state = volume_test_state(&tmp);
|
||||
let volume = state
|
||||
.store
|
||||
.read()
|
||||
.unwrap()
|
||||
.find_volume(VolumeId(1))
|
||||
.unwrap()
|
||||
.1;
|
||||
let payload = "compressible replica text ".repeat(64);
|
||||
for (id, query, compressed) in [
|
||||
(0x6e7a_0c08u64, "?type=replicate", false),
|
||||
(0x6e7a_0c09u64, "", true),
|
||||
] {
|
||||
let response = super::super::volume_server::build_admin_router(state.clone())
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method(Method::POST)
|
||||
.uri(format!("/1,{id:x}{TEST_COOKIE:08x}{query}"))
|
||||
.header(header::CONTENT_TYPE, "text/plain")
|
||||
.body(Body::from(payload.clone()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::CREATED);
|
||||
let mut stored = Needle {
|
||||
id: NeedleId(id),
|
||||
cookie: Cookie(TEST_COOKIE),
|
||||
..Needle::default()
|
||||
};
|
||||
volume.read_needle(&mut stored).unwrap();
|
||||
assert_eq!(stored.is_compressed(), compressed, "query {query:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_orientation_fix_follows_the_filename() {
|
||||
use tower::ServiceExt;
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let mut state = volume_test_state(&tmp);
|
||||
Arc::get_mut(&mut state).unwrap().fix_jpg_orientation = true;
|
||||
let volume = state
|
||||
.store
|
||||
.read()
|
||||
.unwrap()
|
||||
.find_volume(VolumeId(1))
|
||||
.unwrap()
|
||||
.1;
|
||||
let mut encoded = std::io::Cursor::new(Vec::new());
|
||||
image::DynamicImage::ImageRgba8(image::RgbaImage::new(3, 2))
|
||||
.write_to(&mut encoded, image::ImageFormat::Jpeg)
|
||||
.unwrap();
|
||||
let exif: &[u8] =
|
||||
b"Exif\0\0MM\0\x2a\0\0\0\x08\0\x01\x01\x12\0\x03\0\0\0\x01\0\x03\0\0\0\0\0\0";
|
||||
let mut tagged = vec![0xFF, 0xD8, 0xFF, 0xE1, 0, exif.len() as u8 + 2];
|
||||
tagged.extend_from_slice(exif);
|
||||
tagged.extend_from_slice(&encoded.into_inner()[2..]);
|
||||
let upload = |id: u64, suffix: &'static str, filename: &'static str, body: Vec<u8>| {
|
||||
let (state, volume) = (state.clone(), volume.clone());
|
||||
async move {
|
||||
let response = super::super::volume_server::build_admin_router(state)
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method(Method::POST)
|
||||
.uri(format!("/1,{id:x}{TEST_COOKIE:08x}{suffix}"))
|
||||
.header(header::CONTENT_TYPE, "application/octet-stream")
|
||||
.header(
|
||||
header::CONTENT_DISPOSITION,
|
||||
format!("attachment; filename=\"{filename}\""),
|
||||
)
|
||||
.body(Body::from(body))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
StatusCode::CREATED,
|
||||
"{suffix} {filename}"
|
||||
);
|
||||
let mut stored = Needle {
|
||||
id: NeedleId(id),
|
||||
cookie: Cookie(TEST_COOKIE),
|
||||
..Needle::default()
|
||||
};
|
||||
volume.read_needle(&mut stored).unwrap();
|
||||
stored.data
|
||||
}
|
||||
};
|
||||
let rotated = upload(0x6e7a_0c10, "", "photo.jpg", tagged.clone()).await;
|
||||
assert_ne!(rotated, tagged);
|
||||
assert_eq!(
|
||||
upload(0x6e7a_0c11, ".jpg", "photo.png", tagged.clone()).await,
|
||||
tagged
|
||||
);
|
||||
assert_eq!(
|
||||
upload(0x6e7a_0c12, "?type=replicate", "photo.jpg", tagged.clone()).await,
|
||||
rotated
|
||||
);
|
||||
assert_eq!(
|
||||
upload(0x6e7a_0c13, "?type=replicate", "photo.jpg", rotated.clone()).await,
|
||||
rotated
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_multipart_replica_refusal_is_not_marked() {
|
||||
use tower::ServiceExt;
|
||||
|
||||
@@ -9,14 +9,14 @@
|
||||
//! Matches Go's storage/volume.go, volume_loading.go, volume_read.go,
|
||||
//! volume_write.go, volume_super_block.go.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, Read, Seek, SeekFrom, Write};
|
||||
use std::ops::ControlFlow;
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU8, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Condvar, Mutex, RwLock};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use std::sync::{Arc, Condvar, LazyLock, Mutex, RwLock};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
@@ -1198,6 +1198,33 @@ impl RemoteDatFile {
|
||||
.backend
|
||||
.read_range_blocking(&self.key, offset, buf.len())
|
||||
.map_err(io::Error::other)?;
|
||||
Self::fill(buf, offset, data)
|
||||
}
|
||||
|
||||
/// `read_exact_at` that fails with `TimedOut` once `timeout` passes.
|
||||
pub(crate) fn read_exact_at_within(
|
||||
&self,
|
||||
buf: &mut [u8],
|
||||
offset: u64,
|
||||
timeout: Duration,
|
||||
) -> io::Result<()> {
|
||||
match self
|
||||
.backend
|
||||
.read_range_blocking_within(&self.key, offset, buf.len(), timeout)
|
||||
.map_err(io::Error::other)?
|
||||
{
|
||||
Some(data) => Self::fill(buf, offset, data),
|
||||
None => Err(io::Error::new(
|
||||
io::ErrorKind::TimedOut,
|
||||
format!(
|
||||
"remote read at offset {} got no answer within {:?}",
|
||||
offset, timeout
|
||||
),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn fill(buf: &mut [u8], offset: u64, data: Vec<u8>) -> io::Result<()> {
|
||||
if data.len() != buf.len() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
@@ -1214,6 +1241,71 @@ impl RemoteDatFile {
|
||||
}
|
||||
}
|
||||
|
||||
/// A tiered volume loads without its superblock when the backend cannot serve
|
||||
/// it; once a read stalls, the backend's other volumes skip it for a while.
|
||||
/// Matches Go's remoteSuperBlockTimeout and remoteSuperBlockStallPeriod.
|
||||
const REMOTE_SUPER_BLOCK_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const REMOTE_SUPER_BLOCK_STALL_PERIOD: Duration = Duration::from_secs(60);
|
||||
static REMOTE_SUPER_BLOCK_STALLS: LazyLock<Mutex<HashMap<String, Instant>>> =
|
||||
LazyLock::new(Default::default);
|
||||
|
||||
fn read_super_block_bytes(
|
||||
mut read: impl FnMut(&mut [u8], u64) -> Result<(), VolumeError>,
|
||||
) -> Result<Vec<u8>, VolumeError> {
|
||||
let mut header = [0u8; SUPER_BLOCK_SIZE];
|
||||
read(&mut header, 0)?;
|
||||
|
||||
let extra_size = u16::from_be_bytes([header[6], header[7]]);
|
||||
let mut full_buf = vec![0u8; SUPER_BLOCK_SIZE + extra_size as usize];
|
||||
full_buf[..SUPER_BLOCK_SIZE].copy_from_slice(&header);
|
||||
if extra_size > 0 {
|
||||
read(&mut full_buf[SUPER_BLOCK_SIZE..], SUPER_BLOCK_SIZE as u64)?;
|
||||
}
|
||||
Ok(full_buf)
|
||||
}
|
||||
|
||||
fn read_remote_super_block(
|
||||
storage_name: &str,
|
||||
remote: &RemoteDatFile,
|
||||
timeout: Duration,
|
||||
stall_period: Duration,
|
||||
) -> Result<Vec<u8>, VolumeError> {
|
||||
if let Some(until) = REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().get(storage_name)
|
||||
&& Instant::now() < *until
|
||||
{
|
||||
return Err(VolumeError::Io(io::Error::other(format!(
|
||||
"backend {} stalled on a super block read, skipped",
|
||||
storage_name
|
||||
))));
|
||||
}
|
||||
let deadline = Instant::now() + timeout;
|
||||
let result = read_super_block_bytes(|buf, offset| {
|
||||
remote
|
||||
.read_exact_at_within(
|
||||
buf,
|
||||
offset,
|
||||
deadline.saturating_duration_since(Instant::now()),
|
||||
)
|
||||
.map_err(VolumeError::from)
|
||||
});
|
||||
match &result {
|
||||
Ok(_) => {
|
||||
REMOTE_SUPER_BLOCK_STALLS
|
||||
.lock()
|
||||
.unwrap()
|
||||
.remove(storage_name);
|
||||
}
|
||||
Err(VolumeError::Io(e)) if e.kind() == io::ErrorKind::TimedOut => {
|
||||
REMOTE_SUPER_BLOCK_STALLS
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(storage_name.to_string(), Instant::now() + stall_period);
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Volume
|
||||
// ============================================================================
|
||||
@@ -1684,6 +1776,7 @@ impl Volume {
|
||||
)));
|
||||
}
|
||||
|
||||
let mut remote_super_block_failed = false;
|
||||
if already_has_super_block {
|
||||
match self.read_super_block(io) {
|
||||
Ok(()) => {
|
||||
@@ -1700,6 +1793,12 @@ impl Volume {
|
||||
error = %e,
|
||||
"failed to read remote super block during load"
|
||||
);
|
||||
remote_super_block_failed = true;
|
||||
let replication = self.volume_info.read().unwrap().replication.clone();
|
||||
if !replication.is_empty() {
|
||||
self.super_block.write().unwrap().replica_placement =
|
||||
ReplicaPlacement::from_string(&replication)?;
|
||||
}
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
@@ -1748,17 +1847,24 @@ impl Volume {
|
||||
// at the front of .idx. Best effort: a volume that cannot be
|
||||
// repaired is still servable for everything the surviving rows
|
||||
// index.
|
||||
match self.repair_idx_head_tombstones(io) {
|
||||
Ok(0) => {}
|
||||
Ok(restored) => info!(
|
||||
if remote_super_block_failed {
|
||||
debug!(
|
||||
volume_id = self.id.0,
|
||||
restored, "recovered overwritten .idx rows from .dat"
|
||||
),
|
||||
Err(e) => warn!(
|
||||
volume_id = self.id.0,
|
||||
error = %e,
|
||||
"recover overwritten .idx rows"
|
||||
),
|
||||
"remote backend unavailable, not recovering overwritten .idx rows on this load"
|
||||
);
|
||||
} else {
|
||||
match self.repair_idx_head_tombstones(io) {
|
||||
Ok(0) => {}
|
||||
Ok(restored) => info!(
|
||||
volume_id = self.id.0,
|
||||
restored, "recovered overwritten .idx rows from .dat"
|
||||
),
|
||||
Err(e) => warn!(
|
||||
volume_id = self.id.0,
|
||||
error = %e,
|
||||
"recover overwritten .idx rows"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// The permission bits still allow writing on a read-only mount;
|
||||
@@ -2220,21 +2326,15 @@ impl Volume {
|
||||
// ---- SuperBlock I/O ----
|
||||
|
||||
fn read_super_block(&self, io: &VolumeIo) -> Result<(), VolumeError> {
|
||||
let mut header = [0u8; SUPER_BLOCK_SIZE];
|
||||
self.read_exact_at_backend(io, &mut header, 0)?;
|
||||
|
||||
let extra_size = u16::from_be_bytes([header[6], header[7]]);
|
||||
let total_size = SUPER_BLOCK_SIZE + extra_size as usize;
|
||||
|
||||
let mut full_buf = vec![0u8; total_size];
|
||||
full_buf[..SUPER_BLOCK_SIZE].copy_from_slice(&header);
|
||||
if extra_size > 0 {
|
||||
self.read_exact_at_backend(
|
||||
io,
|
||||
&mut full_buf[SUPER_BLOCK_SIZE..],
|
||||
SUPER_BLOCK_SIZE as u64,
|
||||
)?;
|
||||
}
|
||||
let full_buf = match io.remote_dat_file.as_ref() {
|
||||
Some(remote) if self.has_remote_file() => read_remote_super_block(
|
||||
&self.remote_storage_name_key().0,
|
||||
remote,
|
||||
REMOTE_SUPER_BLOCK_TIMEOUT,
|
||||
REMOTE_SUPER_BLOCK_STALL_PERIOD,
|
||||
)?,
|
||||
_ => read_super_block_bytes(|buf, offset| self.read_exact_at_backend(io, buf, offset))?,
|
||||
};
|
||||
|
||||
*self.super_block.write().unwrap() = SuperBlock::from_bytes(&full_buf)?;
|
||||
|
||||
@@ -11523,6 +11623,254 @@ mod tests {
|
||||
assert!(result.is_err(), "unopenable .dat must fail load");
|
||||
}
|
||||
|
||||
fn silent_tier_backend() -> (
|
||||
std::net::TcpListener,
|
||||
Arc<crate::remote_storage::s3_tier::S3TierBackend>,
|
||||
) {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let backend = crate::remote_storage::s3_tier::S3TierBackend::new(
|
||||
&crate::remote_storage::s3_tier::S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
bucket: "bucket-a".to_string(),
|
||||
endpoint: format!("http://{}", listener.local_addr().unwrap()),
|
||||
storage_class: "STANDARD".to_string(),
|
||||
force_path_style: true,
|
||||
},
|
||||
);
|
||||
(listener, Arc::new(backend))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_super_block_read_gives_up_on_a_silent_backend_and_skips_it() {
|
||||
let name = "s3.super_block_stall_test";
|
||||
let (_listener, backend) = silent_tier_backend();
|
||||
let silent = RemoteDatFile {
|
||||
backend,
|
||||
key: "remote-key".to_string(),
|
||||
file_size: 1024,
|
||||
modified_time: 0,
|
||||
};
|
||||
|
||||
let start = Instant::now();
|
||||
let err = read_remote_super_block(
|
||||
name,
|
||||
&silent,
|
||||
Duration::from_millis(200),
|
||||
Duration::from_secs(3600),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(&err, VolumeError::Io(e) if e.kind() == io::ErrorKind::TimedOut),
|
||||
"{err:?}"
|
||||
);
|
||||
assert!(start.elapsed() < Duration::from_secs(5));
|
||||
|
||||
let start = Instant::now();
|
||||
assert!(
|
||||
read_remote_super_block(
|
||||
name,
|
||||
&silent,
|
||||
Duration::from_secs(30),
|
||||
Duration::from_secs(3600)
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
start.elapsed() < Duration::from_secs(1),
|
||||
"a stalled backend must not be asked again within the stall period"
|
||||
);
|
||||
|
||||
let sb = SuperBlock {
|
||||
version: Version::current(),
|
||||
..SuperBlock::default()
|
||||
};
|
||||
let (endpoint, shutdown_tx) = spawn_fake_s3_server(sb.to_bytes());
|
||||
let answering = RemoteDatFile {
|
||||
backend: Arc::new(crate::remote_storage::s3_tier::S3TierBackend::new(
|
||||
&crate::remote_storage::s3_tier::S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
bucket: "bucket-a".to_string(),
|
||||
endpoint,
|
||||
storage_class: "STANDARD".to_string(),
|
||||
force_path_style: true,
|
||||
},
|
||||
)),
|
||||
key: "remote-key".to_string(),
|
||||
file_size: SUPER_BLOCK_SIZE as u64,
|
||||
modified_time: 0,
|
||||
};
|
||||
REMOTE_SUPER_BLOCK_STALLS
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(name.to_string(), Instant::now());
|
||||
let bytes = read_remote_super_block(
|
||||
name,
|
||||
&answering,
|
||||
Duration::from_secs(10),
|
||||
Duration::from_secs(3600),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
SuperBlock::from_bytes(&bytes).unwrap().version,
|
||||
Version::current()
|
||||
);
|
||||
assert!(
|
||||
!REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().contains_key(name),
|
||||
"a successful read clears the stall"
|
||||
);
|
||||
let _ = shutdown_tx.send(());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_volume_loads_without_waiting_on_a_stalled_backend() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let dat_len = {
|
||||
let v = make_test_volume(dir);
|
||||
v.dat_file_size().unwrap()
|
||||
};
|
||||
std::fs::remove_file(format!("{}/1.dat", dir)).unwrap();
|
||||
|
||||
let (_listener, backend) = silent_tier_backend();
|
||||
let name = "s3.load_stall_test";
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.register(name.to_string(), Arc::try_unwrap(backend).ok().unwrap());
|
||||
REMOTE_SUPER_BLOCK_STALLS
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(name.to_string(), Instant::now() + Duration::from_secs(3600));
|
||||
|
||||
let vif = VifVolumeInfo {
|
||||
files: vec![VifRemoteFile {
|
||||
backend_type: "s3".to_string(),
|
||||
backend_id: "load_stall_test".to_string(),
|
||||
key: "remote-key".to_string(),
|
||||
offset: 0,
|
||||
file_size: dat_len,
|
||||
modified_time: 123,
|
||||
extension: ".dat".to_string(),
|
||||
}],
|
||||
version: Version::current().0 as u32,
|
||||
replication: "001".to_string(),
|
||||
..VifVolumeInfo::default()
|
||||
};
|
||||
std::fs::write(
|
||||
format!("{}/1.vif", dir),
|
||||
serde_json::to_string_pretty(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let start = Instant::now();
|
||||
let v = Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
VolumeId(1),
|
||||
NeedleMapKind::InMemory,
|
||||
&VolumeSpec::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(start.elapsed() < Duration::from_secs(5));
|
||||
assert!(v.has_remote_file());
|
||||
assert_eq!(v.version(), Version::current());
|
||||
assert_eq!(
|
||||
v.super_block.read().unwrap().replica_placement,
|
||||
ReplicaPlacement::from_string("001").unwrap()
|
||||
);
|
||||
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.remove(name);
|
||||
REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().remove(name);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_volume_load_skips_idx_repair_on_a_stalled_backend() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let dat_len = {
|
||||
let v = make_test_volume(dir);
|
||||
for i in 1..=3u64 {
|
||||
let mut n = Needle {
|
||||
id: NeedleId(i),
|
||||
cookie: Cookie(0x1234),
|
||||
data: format!("needle-{i}").into_bytes(),
|
||||
data_size: format!("needle-{i}").len() as u32,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true, false).unwrap();
|
||||
}
|
||||
v.dat_file_size().unwrap()
|
||||
};
|
||||
std::fs::remove_file(format!("{}/1.dat", dir)).unwrap();
|
||||
let idx_path = format!("{}/1.idx", dir);
|
||||
crate::storage::volume_idx_repair::tests::clobber_idx_head(&idx_path, &[4, 5]);
|
||||
let idx_before = std::fs::read(&idx_path).unwrap();
|
||||
|
||||
let (_listener, backend) = silent_tier_backend();
|
||||
let name = "s3.idx_repair_stall_test";
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.register(name.to_string(), Arc::try_unwrap(backend).ok().unwrap());
|
||||
REMOTE_SUPER_BLOCK_STALLS
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(name.to_string(), Instant::now() + Duration::from_secs(3600));
|
||||
|
||||
let vif = VifVolumeInfo {
|
||||
files: vec![VifRemoteFile {
|
||||
backend_type: "s3".to_string(),
|
||||
backend_id: "idx_repair_stall_test".to_string(),
|
||||
key: "remote-key".to_string(),
|
||||
offset: 0,
|
||||
file_size: dat_len,
|
||||
modified_time: 123,
|
||||
extension: ".dat".to_string(),
|
||||
}],
|
||||
version: Version::current().0 as u32,
|
||||
..VifVolumeInfo::default()
|
||||
};
|
||||
std::fs::write(
|
||||
format!("{}/1.vif", dir),
|
||||
serde_json::to_string_pretty(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let start = Instant::now();
|
||||
let v = Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
VolumeId(1),
|
||||
NeedleMapKind::InMemory,
|
||||
&VolumeSpec::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
start.elapsed() < Duration::from_secs(5),
|
||||
"the load waited on the stalled backend"
|
||||
);
|
||||
assert!(v.has_remote_file());
|
||||
drop(v);
|
||||
assert_eq!(
|
||||
std::fs::read(&idx_path).unwrap(),
|
||||
idx_before,
|
||||
"the repair ran against an unreachable backend"
|
||||
);
|
||||
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.remove(name);
|
||||
REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().remove(name);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_remote_only_volume_load_reads_from_tier_backend() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
|
||||
@@ -224,7 +224,7 @@ fn finish_walk(walk: io::Result<()>) -> Result<(), VolumeError> {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
use crate::storage::needle::crc::CRC;
|
||||
use crate::storage::needle_map::NeedleMapKind;
|
||||
@@ -267,7 +267,7 @@ mod tests {
|
||||
/// Reproduce the damage a delete on a tiered read-only volume used to do: it
|
||||
/// writes (key, offset 0, tombstone) rows over the front of .idx instead of
|
||||
/// appending them.
|
||||
fn clobber_idx_head(idx_path: &str, keys: &[u64]) {
|
||||
pub(crate) fn clobber_idx_head(idx_path: &str, keys: &[u64]) {
|
||||
let mut file = OpenOptions::new().write(true).open(idx_path).unwrap();
|
||||
for (i, key) in keys.iter().enumerate() {
|
||||
let mut row = Vec::new();
|
||||
|
||||
@@ -95,6 +95,11 @@ gateway without STS configured vends no credentials at all, so `--access-key`
|
||||
and `--secret-key` supply a fallback; anything the namespace does vend wins over
|
||||
them.
|
||||
|
||||
In a vector bucket's `vector` namespace the gateway vends write credentials only
|
||||
to a caller that proves it holds `jwt.filer_signing.key`. Pass that key with
|
||||
`--filer-signing-key` so compaction, cleanup and index builds can rewrite
|
||||
vector indexes.
|
||||
|
||||
## State
|
||||
|
||||
All three jobs are implemented and tested end to end against a live gateway:
|
||||
|
||||
@@ -33,9 +33,26 @@ pub fn admin_token(signing_key: &[u8], expires_after_secs: u64) -> Option<String
|
||||
))
|
||||
}
|
||||
|
||||
/// Derives the key for one use of a signing key, so a token signed with it is
|
||||
/// not accepted where the signing key itself is.
|
||||
pub fn derive_key(signing_key: &[u8], purpose: &str) -> Vec<u8> {
|
||||
let key = hmac::Key::new(hmac::HMAC_SHA256, signing_key);
|
||||
hmac::sign(&key, purpose.as_bytes()).as_ref().to_vec()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::admin_token;
|
||||
use super::{admin_token, derive_key};
|
||||
|
||||
#[test]
|
||||
fn derive_key_matches_the_go_derivation() {
|
||||
let derived = derive_key(b"cluster-key", "seaweedfs-lance-cluster-token");
|
||||
let hex: String = derived.iter().map(|b| format!("{b:02x}")).collect();
|
||||
assert_eq!(
|
||||
hex,
|
||||
"b38c0b66da41621f2a9b6de68e4239ddd0d2dd7b4ede01aa91bd34a17edc1021"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_key_no_token() {
|
||||
|
||||
@@ -7,4 +7,4 @@
|
||||
|
||||
pub mod namespace;
|
||||
|
||||
pub use namespace::{NamespaceClient, TableDescription, parse_id};
|
||||
pub use namespace::{NamespaceClient, SigningKey, TableDescription, parse_id};
|
||||
@@ -8,6 +8,39 @@ use serde::Deserialize;
|
||||
/// The delimiter the Lance namespace joins identifier parts with.
|
||||
const DELIMITER: &str = "$";
|
||||
|
||||
/// Carries a token that proves the caller holds the cluster key, which marks
|
||||
/// it as one of the cluster's own components.
|
||||
const CLUSTER_TOKEN_HEADER: &str = "X-Seaweedfs-Cluster-Token";
|
||||
|
||||
/// A cluster token is minted for each request, so it only has to outlast the
|
||||
/// clock skew between this host and the gateway.
|
||||
const CLUSTER_TOKEN_SECS: u64 = 300;
|
||||
|
||||
/// The key cluster tokens are signed with, derived from the cluster key
|
||||
/// (`jwt.filer_signing.key`) the way the gateway derives it, so a token
|
||||
/// captured on the wire does not authorize admin calls. The catalog vends
|
||||
/// credentials that write a vector bucket's index data only to callers that
|
||||
/// send one. Not Debug, so the key cannot reach a log.
|
||||
#[derive(Clone)]
|
||||
pub struct SigningKey {
|
||||
key: Vec<u8>,
|
||||
}
|
||||
|
||||
impl SigningKey {
|
||||
pub fn new(key: Option<String>) -> Option<Self> {
|
||||
key.filter(|key| !key.is_empty()).map(|key| Self {
|
||||
key: seaweed_worker_core::token::derive_key(
|
||||
key.as_bytes(),
|
||||
"seaweedfs-lance-cluster-token",
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
fn token(&self) -> Option<String> {
|
||||
seaweed_worker_core::token::admin_token(&self.key, CLUSTER_TOKEN_SECS)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct TableDescription {
|
||||
pub location: String,
|
||||
@@ -35,6 +68,7 @@ pub struct NamespaceClient {
|
||||
base_url: String,
|
||||
http: reqwest::Client,
|
||||
api_key: Option<String>,
|
||||
signing_key: Option<SigningKey>,
|
||||
}
|
||||
|
||||
/// A namespace call that has not answered by now is not going to. Without this
|
||||
@@ -71,6 +105,7 @@ impl NamespaceClient {
|
||||
base_url: base_url.into().trim_end_matches('/').to_string(),
|
||||
http,
|
||||
api_key: None,
|
||||
signing_key: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,10 +127,25 @@ impl NamespaceClient {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_signing_key(mut self, signing_key: Option<&SigningKey>) -> Self {
|
||||
if signing_key.is_some() && !protected_transport(&self.base_url) {
|
||||
tracing::warn!(
|
||||
"catalog url {} exposes the cluster token in cleartext",
|
||||
self.base_url
|
||||
);
|
||||
}
|
||||
self.signing_key = signing_key.cloned();
|
||||
self
|
||||
}
|
||||
|
||||
fn authed(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
||||
match &self.api_key {
|
||||
let request = match &self.api_key {
|
||||
Some(key) => request.header("x-api-key", key),
|
||||
None => request,
|
||||
};
|
||||
match self.signing_key.as_ref().and_then(SigningKey::token) {
|
||||
Some(token) => request.header(CLUSTER_TOKEN_HEADER, token),
|
||||
None => request,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -224,7 +274,74 @@ pub fn parse_id(encoded: &str) -> Vec<String> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{MAX_REDIRECT_HOPS, may_follow, protected_transport, same_origin};
|
||||
use super::{
|
||||
CLUSTER_TOKEN_SECS, MAX_REDIRECT_HOPS, NamespaceClient, SigningKey, may_follow,
|
||||
protected_transport, same_origin,
|
||||
};
|
||||
|
||||
/// A catalog that answers one request with a table location and hands back
|
||||
/// the request it read.
|
||||
async fn recording_catalog() -> (String, tokio::sync::oneshot::Receiver<String>) {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let (sender, receiver) = tokio::sync::oneshot::channel();
|
||||
tokio::spawn(async move {
|
||||
let (mut conn, _) = listener.accept().await.unwrap();
|
||||
let mut request = vec![0u8; 8192];
|
||||
let n = conn.read(&mut request).await.unwrap();
|
||||
let _ = sender.send(String::from_utf8_lossy(&request[..n]).into_owned());
|
||||
let body = r#"{"location":"s3://b/vector/t"}"#;
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
let _ = conn.write_all(response.as_bytes()).await;
|
||||
});
|
||||
(url, receiver)
|
||||
}
|
||||
|
||||
fn header<'a>(request: &'a str, name: &str) -> Option<&'a str> {
|
||||
request.lines().find_map(|line| {
|
||||
let (key, value) = line.split_once(':')?;
|
||||
key.eq_ignore_ascii_case(name).then(|| value.trim())
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn signing_key_sends_a_cluster_token() {
|
||||
let id = ["b".to_string(), "vector".to_string(), "t".to_string()];
|
||||
let (url, request) = recording_catalog().await;
|
||||
let key = SigningKey::new(Some("cluster-key".into()));
|
||||
NamespaceClient::new(url)
|
||||
.with_signing_key(key.as_ref())
|
||||
.describe_table(&id)
|
||||
.await
|
||||
.unwrap();
|
||||
let request = request.await.unwrap();
|
||||
let token = header(&request, "x-seaweedfs-cluster-token").expect("cluster token");
|
||||
let raw =
|
||||
seaweed_worker_core::token::admin_token(b"cluster-key", CLUSTER_TOKEN_SECS).unwrap();
|
||||
assert_eq!(token.split('.').count(), 3);
|
||||
assert_ne!(
|
||||
token.rsplit('.').next(),
|
||||
raw.rsplit('.').next(),
|
||||
"signed with the cluster key itself"
|
||||
);
|
||||
|
||||
let (url, request) = recording_catalog().await;
|
||||
NamespaceClient::new(url).describe_table(&id).await.unwrap();
|
||||
assert_eq!(
|
||||
header(&request.await.unwrap(), "x-seaweedfs-cluster-token"),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_signing_key_is_no_key() {
|
||||
assert!(SigningKey::new(None).is_none());
|
||||
assert!(SigningKey::new(Some(String::new())).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn may_follow_bounds_hops_within_the_origin() {
|
||||
|
||||
@@ -65,21 +65,38 @@ pub async fn open_with(
|
||||
})
|
||||
}
|
||||
|
||||
/// The namespace vends object_store's own option names, so they pass straight
|
||||
/// through. What it vends always wins: the fallback exists because a deployment
|
||||
/// without STS vends no credentials at all, and then the worker has no other way
|
||||
/// to reach the data.
|
||||
/// The object_store options for a table. The namespace vends object_store's own
|
||||
/// option names, so they pass straight through. What it vends always wins: the
|
||||
/// fallback exists because a deployment without STS vends no credentials at
|
||||
/// all, and then the worker has no other way to reach the data.
|
||||
pub fn table_options(
|
||||
description: &TableDescription,
|
||||
fallback: &FallbackOptions,
|
||||
) -> HashMap<String, String> {
|
||||
let mut options: HashMap<String, String> = fallback.clone();
|
||||
options.extend(description.storage_options.clone());
|
||||
// A store that takes unsigned requests needs no credentials, but without
|
||||
// any lance builds the AWS default chain, and that probes instance
|
||||
// metadata on every open.
|
||||
if options
|
||||
.get("aws_skip_signature")
|
||||
.is_some_and(|v| v == "true")
|
||||
{
|
||||
options.entry("aws_access_key_id".into()).or_default();
|
||||
options.entry("aws_secret_access_key".into()).or_default();
|
||||
}
|
||||
options
|
||||
}
|
||||
|
||||
async fn open_at(
|
||||
description: &TableDescription,
|
||||
fallback: &FallbackOptions,
|
||||
wrapper: Option<Arc<dyn WrappingObjectStore>>,
|
||||
) -> Result<Dataset> {
|
||||
let mut options: HashMap<String, String> = fallback.clone();
|
||||
options.extend(description.storage_options.clone());
|
||||
DatasetBuilder::from_uri(&description.location)
|
||||
.with_store_params(ObjectStoreParams {
|
||||
storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
|
||||
options,
|
||||
table_options(description, fallback),
|
||||
))),
|
||||
object_store_wrapper: wrapper,
|
||||
..Default::default()
|
||||
@@ -102,14 +119,12 @@ pub async fn drop(
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
check_drop_location(id, &description.location)?;
|
||||
let mut options: HashMap<String, String> = fallback.clone();
|
||||
options.extend(description.storage_options.clone());
|
||||
let (store, path) = ObjectStore::from_uri_and_params(
|
||||
Arc::new(ObjectStoreRegistry::default()),
|
||||
&description.location,
|
||||
&ObjectStoreParams {
|
||||
storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
|
||||
options,
|
||||
table_options(&description, fallback),
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -203,12 +218,45 @@ fn schema_json(dataset: &Dataset) -> Option<String> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{check_drop_location, table_location};
|
||||
use super::{FallbackOptions, check_drop_location, table_location, table_options};
|
||||
use crate::catalog::TableDescription;
|
||||
|
||||
fn id(parts: &[&str]) -> Vec<String> {
|
||||
parts.iter().map(|part| part.to_string()).collect()
|
||||
}
|
||||
|
||||
fn options(pairs: &[(&str, &str)]) -> FallbackOptions {
|
||||
pairs
|
||||
.iter()
|
||||
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_options_keep_an_unsigned_store_off_the_aws_default_chain() {
|
||||
let described = |vended: &[(&str, &str)]| TableDescription {
|
||||
location: "s3://bkt/vector/x".into(),
|
||||
storage_options: options(vended),
|
||||
managed_versioning: false,
|
||||
};
|
||||
let unsigned = described(&[("aws_skip_signature", "true")]);
|
||||
|
||||
let got = table_options(&unsigned, &FallbackOptions::new());
|
||||
assert_eq!(got.get("aws_access_key_id").map(String::as_str), Some(""));
|
||||
assert_eq!(
|
||||
got.get("aws_secret_access_key").map(String::as_str),
|
||||
Some("")
|
||||
);
|
||||
|
||||
let fallback = options(&[("aws_access_key_id", "k"), ("aws_secret_access_key", "s")]);
|
||||
let got = table_options(&unsigned, &fallback);
|
||||
assert_eq!(got.get("aws_access_key_id").map(String::as_str), Some("k"));
|
||||
|
||||
let signed = described(&[("aws_endpoint", "http://s3:8333")]);
|
||||
let got = table_options(&signed, &FallbackOptions::new());
|
||||
assert!(!got.contains_key("aws_access_key_id"), "{got:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_location_matches_the_gateway_layout() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -13,7 +13,7 @@ use seaweed_worker_core::pb::{
|
||||
use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::catalog::{NamespaceClient, parse_id};
|
||||
use crate::catalog::{NamespaceClient, SigningKey, parse_id};
|
||||
use crate::dataset;
|
||||
use crate::jobs::{clamp, string_list, table_id};
|
||||
|
||||
@@ -32,6 +32,7 @@ const MAX_MIN_VERSIONS: i64 = 1000;
|
||||
pub struct CleanupVersionsHandler {
|
||||
namespace_url: String,
|
||||
fallback: dataset::FallbackOptions,
|
||||
signing_key: Option<SigningKey>,
|
||||
metrics: Option<crate::metrics::LanceMetrics>,
|
||||
}
|
||||
|
||||
@@ -40,6 +41,7 @@ impl CleanupVersionsHandler {
|
||||
Self {
|
||||
namespace_url,
|
||||
fallback: dataset::FallbackOptions::new(),
|
||||
signing_key: None,
|
||||
metrics: None,
|
||||
}
|
||||
}
|
||||
@@ -54,8 +56,15 @@ impl CleanupVersionsHandler {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_signing_key(mut self, signing_key: Option<SigningKey>) -> Self {
|
||||
self.signing_key = signing_key;
|
||||
self
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
.with_fallback(&self.fallback)
|
||||
.with_signing_key(self.signing_key.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ use seaweed_worker_core::pb::{
|
||||
use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::catalog::{NamespaceClient, parse_id};
|
||||
use crate::catalog::{NamespaceClient, SigningKey, parse_id};
|
||||
use crate::dataset;
|
||||
use crate::jobs::{FORMAT, clamp, observation, string_list, table_id};
|
||||
|
||||
@@ -35,6 +35,7 @@ const MIN_FRAGMENTS_CEILING: i64 = 4096;
|
||||
pub struct CompactHandler {
|
||||
namespace_url: String,
|
||||
fallback: dataset::FallbackOptions,
|
||||
signing_key: Option<SigningKey>,
|
||||
metrics: Option<crate::metrics::LanceMetrics>,
|
||||
}
|
||||
|
||||
@@ -43,6 +44,7 @@ impl CompactHandler {
|
||||
Self {
|
||||
namespace_url,
|
||||
fallback: dataset::FallbackOptions::new(),
|
||||
signing_key: None,
|
||||
metrics: None,
|
||||
}
|
||||
}
|
||||
@@ -58,8 +60,15 @@ impl CompactHandler {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_signing_key(mut self, signing_key: Option<SigningKey>) -> Self {
|
||||
self.signing_key = signing_key;
|
||||
self
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
.with_fallback(&self.fallback)
|
||||
.with_signing_key(self.signing_key.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ use seaweed_worker_core::pb::{
|
||||
use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::catalog::{NamespaceClient, parse_id};
|
||||
use crate::catalog::{NamespaceClient, SigningKey, parse_id};
|
||||
use crate::dataset::{self, OpenTable};
|
||||
use crate::jobs::{clamp, string_list, table_id};
|
||||
|
||||
@@ -35,6 +35,7 @@ const MAX_UNINDEXED_CEILING: i64 = 100_000_000;
|
||||
pub struct OptimizeIndicesHandler {
|
||||
namespace_url: String,
|
||||
fallback: dataset::FallbackOptions,
|
||||
signing_key: Option<SigningKey>,
|
||||
metrics: Option<crate::metrics::LanceMetrics>,
|
||||
}
|
||||
|
||||
@@ -43,6 +44,7 @@ impl OptimizeIndicesHandler {
|
||||
Self {
|
||||
namespace_url,
|
||||
fallback: dataset::FallbackOptions::new(),
|
||||
signing_key: None,
|
||||
metrics: None,
|
||||
}
|
||||
}
|
||||
@@ -57,8 +59,15 @@ impl OptimizeIndicesHandler {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_signing_key(mut self, signing_key: Option<SigningKey>) -> Self {
|
||||
self.signing_key = signing_key;
|
||||
self
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
.with_fallback(&self.fallback)
|
||||
.with_signing_key(self.signing_key.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -42,27 +42,32 @@ pub(crate) fn table_id(parameters: &HashMap<String, ConfigValue>) -> Option<Vec<
|
||||
pub fn handlers(
|
||||
namespace_url: String,
|
||||
fallback: crate::dataset::FallbackOptions,
|
||||
signing_key: Option<crate::catalog::SigningKey>,
|
||||
metrics: Option<crate::metrics::LanceMetrics>,
|
||||
) -> Vec<Arc<dyn JobHandler>> {
|
||||
vec![
|
||||
Arc::new(
|
||||
compact::CompactHandler::new(namespace_url.clone())
|
||||
.with_fallback(fallback.clone())
|
||||
.with_signing_key(signing_key.clone())
|
||||
.with_metrics(metrics.clone()),
|
||||
),
|
||||
Arc::new(
|
||||
indices::OptimizeIndicesHandler::new(namespace_url.clone())
|
||||
.with_fallback(fallback.clone())
|
||||
.with_signing_key(signing_key.clone())
|
||||
.with_metrics(metrics.clone()),
|
||||
),
|
||||
Arc::new(
|
||||
sort::SortHandler::new(namespace_url.clone())
|
||||
.with_fallback(fallback.clone())
|
||||
.with_signing_key(signing_key.clone())
|
||||
.with_metrics(metrics.clone()),
|
||||
),
|
||||
Arc::new(
|
||||
cleanup::CleanupVersionsHandler::new(namespace_url)
|
||||
.with_fallback(fallback)
|
||||
.with_signing_key(signing_key)
|
||||
.with_metrics(metrics),
|
||||
),
|
||||
]
|
||||
|
||||
@@ -23,7 +23,7 @@ use seaweed_worker_sort::{
|
||||
};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::catalog::{NamespaceClient, parse_id};
|
||||
use crate::catalog::{NamespaceClient, SigningKey, parse_id};
|
||||
use crate::dataset;
|
||||
use crate::jobs::{FORMAT, clamp, observation, string_list, table_id};
|
||||
|
||||
@@ -54,6 +54,7 @@ const BYTES_PER_MB: u64 = 1024 * 1024;
|
||||
pub struct SortHandler {
|
||||
namespace_url: String,
|
||||
fallback: dataset::FallbackOptions,
|
||||
signing_key: Option<SigningKey>,
|
||||
metrics: Option<crate::metrics::LanceMetrics>,
|
||||
}
|
||||
|
||||
@@ -62,6 +63,7 @@ impl SortHandler {
|
||||
Self {
|
||||
namespace_url,
|
||||
fallback: dataset::FallbackOptions::new(),
|
||||
signing_key: None,
|
||||
metrics: None,
|
||||
}
|
||||
}
|
||||
@@ -77,8 +79,15 @@ impl SortHandler {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_signing_key(mut self, signing_key: Option<SigningKey>) -> Self {
|
||||
self.signing_key = signing_key;
|
||||
self
|
||||
}
|
||||
|
||||
fn client(&self) -> NamespaceClient {
|
||||
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
|
||||
NamespaceClient::new(self.namespace_url.clone())
|
||||
.with_fallback(&self.fallback)
|
||||
.with_signing_key(self.signing_key.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -63,7 +63,8 @@ struct Args {
|
||||
|
||||
/// Shared secret for the admin control stream, the `jwt.filer_signing.key`
|
||||
/// value the Go worker reads from its config file. Required by admins that
|
||||
/// authenticate the worker stream with a signing key.
|
||||
/// authenticate the worker stream with a signing key. The Lance namespace
|
||||
/// takes it too: only a holder gets write credentials to vector indexes.
|
||||
#[arg(long, env = "WEED_FILER_SIGNING_KEY")]
|
||||
filer_signing_key: Option<String>,
|
||||
|
||||
@@ -136,6 +137,7 @@ async fn main() -> Result<()> {
|
||||
|
||||
let args = Args::parse();
|
||||
let tls = args.tls()?;
|
||||
let signing_key = weed_lance_worker::catalog::SigningKey::new(args.filer_signing_key.clone());
|
||||
let options = WorkerOptions {
|
||||
admin_address: args.admin,
|
||||
worker_id: args.id,
|
||||
@@ -164,9 +166,10 @@ async fn main() -> Result<()> {
|
||||
}
|
||||
|
||||
let mut registry = Registry::new().with_preview(Arc::new(
|
||||
weed_lance_worker::preview::LancePreview::new(args.namespace.clone(), fallback.clone()),
|
||||
weed_lance_worker::preview::LancePreview::new(args.namespace.clone(), fallback.clone())
|
||||
.with_signing_key(signing_key.clone()),
|
||||
));
|
||||
for handler in handlers(args.namespace, fallback, Some(lance_metrics)) {
|
||||
for handler in handlers(args.namespace, fallback, signing_key, Some(lance_metrics)) {
|
||||
registry = registry.register(handler);
|
||||
}
|
||||
|
||||
|
||||
@@ -10,13 +10,14 @@ use async_trait::async_trait;
|
||||
use futures::TryStreamExt;
|
||||
use seaweed_worker_core::{Preview, PreviewProvider};
|
||||
|
||||
use crate::catalog::NamespaceClient;
|
||||
use crate::catalog::{NamespaceClient, SigningKey};
|
||||
use crate::dataset::{self, FallbackOptions};
|
||||
use crate::jobs::FORMAT;
|
||||
|
||||
pub struct LancePreview {
|
||||
namespace_url: String,
|
||||
fallback: FallbackOptions,
|
||||
signing_key: Option<SigningKey>,
|
||||
}
|
||||
|
||||
impl LancePreview {
|
||||
@@ -24,8 +25,14 @@ impl LancePreview {
|
||||
Self {
|
||||
namespace_url,
|
||||
fallback,
|
||||
signing_key: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_signing_key(mut self, signing_key: Option<SigningKey>) -> Self {
|
||||
self.signing_key = signing_key;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -35,7 +42,9 @@ impl PreviewProvider for LancePreview {
|
||||
}
|
||||
|
||||
async fn preview(&self, object_id: &[String], row_limit: usize) -> Result<Preview> {
|
||||
let client = NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback);
|
||||
let client = NamespaceClient::new(self.namespace_url.clone())
|
||||
.with_fallback(&self.fallback)
|
||||
.with_signing_key(self.signing_key.as_ref());
|
||||
let table = dataset::open(&client, object_id, &self.fallback).await?;
|
||||
|
||||
let total_rows = table.dataset.count_rows(None).await.unwrap_or(0) as i64;
|
||||
|
||||
@@ -7,6 +7,7 @@ endif
|
||||
VOLUME := ./weed-volume
|
||||
CARGO_PROFILE ?= release
|
||||
CARGO_TARGET_DIR ?= $(abspath ../../seaweed-vector/target)
|
||||
override CARGO_TARGET_DIR := $(abspath $(CARGO_TARGET_DIR))
|
||||
CARGO_OUTPUT := $(CARGO_TARGET_DIR)/$(if $(filter dev,$(CARGO_PROFILE)),debug,$(CARGO_PROFILE))
|
||||
export CARGO_TARGET_DIR
|
||||
|
||||
@@ -18,7 +19,7 @@ $(WEED): FORCE
|
||||
go build -o $(WEED) ../../weed
|
||||
|
||||
$(VEC): FORCE
|
||||
cargo build --locked --profile $(CARGO_PROFILE) --manifest-path ../../seaweed-vector/Cargo.toml -p weed-vector-server
|
||||
cd ../../seaweed-vector && cargo build --locked --profile $(CARGO_PROFILE) -p weed-vector-server
|
||||
cp $(CARGO_OUTPUT)/weed-vector $(VEC)
|
||||
|
||||
ifneq ($(WORKER),)
|
||||
|
||||
@@ -70,8 +70,18 @@ def stop(process):
|
||||
|
||||
@pytest.fixture(scope="module", params=combinations(), ids=lambda p: "-".join(p))
|
||||
def cluster(request, tmp_path_factory):
|
||||
volume, mode = request.param
|
||||
directory = tmp_path_factory.mktemp(f"vectors-{volume}-{mode}")
|
||||
yield from run_cluster(request.param, tmp_path_factory, identities=True)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=combinations(), ids=lambda p: "-".join(p))
|
||||
def open_cluster(request, tmp_path_factory):
|
||||
"""A cluster with no S3 identities, and an engine given no credentials."""
|
||||
yield from run_cluster(request.param, tmp_path_factory, identities=False)
|
||||
|
||||
|
||||
def run_cluster(param, tmp_path_factory, identities):
|
||||
volume, mode = param
|
||||
directory = tmp_path_factory.mktemp(f"vectors-{volume}-{mode}" + ("" if identities else "-open"))
|
||||
print(f"\nVector integration logs: {directory}")
|
||||
weed = binary("WEED_BINARY", "weed")
|
||||
engine = binary("VECTOR_BINARY", "weed-vector")
|
||||
@@ -89,10 +99,14 @@ def cluster(request, tmp_path_factory):
|
||||
master_addr = f"127.0.0.1:{master}.{master_grpc}"
|
||||
filer_addr = f"127.0.0.1:{filer}.{filer_grpc}"
|
||||
|
||||
def start(name, args):
|
||||
def start(name, args, unset=()):
|
||||
log = stack.enter_context((directory / f"{name}.log").open("w"))
|
||||
env = dict(os.environ, AWS_ACCESS_KEY_ID="test", AWS_SECRET_ACCESS_KEY="test",
|
||||
AWS_DEFAULT_REGION="us-east-1", AWS_EC2_METADATA_DISABLED="true")
|
||||
AWS_DEFAULT_REGION="us-east-1", AWS_EC2_METADATA_DISABLED="true",
|
||||
WEED_S3_SSE_KEY="s3vectors-test")
|
||||
env.pop("WEED_S3_SSE_KEK", None)
|
||||
for key in unset:
|
||||
env.pop(key)
|
||||
process = subprocess.Popen(args, cwd=directory, stdout=log, stderr=log, env=env)
|
||||
processes[name] = process
|
||||
stack.callback(stop, process)
|
||||
@@ -117,14 +131,20 @@ def cluster(request, tmp_path_factory):
|
||||
f"-defaultStoreDir={directory / 'filer'}"])
|
||||
wait_for(lambda: listening(filer_grpc), processes, "filer not ready")
|
||||
endpoint = f"http://127.0.0.1:{s3}"
|
||||
start("s3", [weed, "-v=1", "s3", f"-filer={filer_addr}", f"-config={HERE / 's3.json'}",
|
||||
# weed s3 turns AWS_ACCESS_KEY_ID into an admin identity, and the engine
|
||||
# would find it as a credential.
|
||||
if identities:
|
||||
config, unset = [f"-config={HERE / 's3.json'}"], ()
|
||||
credentials = ["--storage-option", "aws_access_key_id=test",
|
||||
"--storage-option", "aws_secret_access_key=test"]
|
||||
else:
|
||||
config, unset, credentials = [], ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"), []
|
||||
start("s3", [weed, "-v=1", "s3", f"-filer={filer_addr}", *config,
|
||||
"-ip=127.0.0.1", "-ip.bind=127.0.0.1", f"-port={s3}",
|
||||
f"-port.grpc={s3_grpc}", "-port.iceberg=0", f"-port.lance={lance}"])
|
||||
f"-port.grpc={s3_grpc}", "-port.iceberg=0", f"-port.lance={lance}"], unset)
|
||||
wait_for(lambda: http_ready(f"http://127.0.0.1:{lance}/v1/namespace/root/list"), processes, "catalog not ready")
|
||||
start("vector", [engine, "--namespace", f"http://127.0.0.1:{lance}",
|
||||
"--masters", master_addr, "--listen", f"127.0.0.1:{vector}",
|
||||
"--storage-option", "aws_access_key_id=test",
|
||||
"--storage-option", "aws_secret_access_key=test"])
|
||||
"--masters", master_addr, "--listen", f"127.0.0.1:{vector}", *credentials], unset)
|
||||
wait_for(lambda: listening(vector), processes, "vector engine not ready")
|
||||
|
||||
wait_for(
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
"""Run the S3 Vectors API against Go/Rust volumes and single/distributed queries."""
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
import pytest
|
||||
|
||||
@@ -58,7 +60,7 @@ def test_vector_lifecycle(vectors):
|
||||
assert [h["key"] for h in hits] == ["a", "c"]
|
||||
distances = {h["key"]: h["distance"] for h in hits}
|
||||
assert distances["a"] == pytest.approx(0.0, abs=1e-6)
|
||||
assert distances["c"] == pytest.approx(0.02, abs=1e-4)
|
||||
assert distances["c"] == pytest.approx(0.02 ** 0.5, abs=1e-4)
|
||||
assert all(h["metadata"]["color"] == "red" for h in hits)
|
||||
|
||||
vectors.delete_vectors(vectorBucketName=BUCKET, indexName=INDEX, keys=["b"])
|
||||
@@ -204,6 +206,32 @@ def test_filter_types(vectors, index):
|
||||
assert sorted(hit["key"] for hit in hits) == ["0", "3"]
|
||||
|
||||
|
||||
def test_filter_arrays(vectors, index):
|
||||
rows = {"space": ["space", "classic"], "nums": [1, 2.5], "strs": ["1", "2.5"], "flag": [True], "scalar": "space"}
|
||||
vectors.put_vectors(**index, vectors=[{"key": key, "data": {"float32": [1, 0, 0, 0]}, "metadata": {"tags": tags}}
|
||||
for key, tags in rows.items()])
|
||||
|
||||
def matches(condition):
|
||||
hits = vectors.query_vectors(**index, queryVector={"float32": [1, 0, 0, 0]}, topK=10,
|
||||
filter={"tags": condition})["vectors"]
|
||||
return sorted(hit["key"] for hit in hits)
|
||||
|
||||
assert matches("space") == ["scalar", "space"]
|
||||
assert matches({"$eq": 1}) == ["nums"]
|
||||
assert matches({"$eq": "1"}) == ["strs"]
|
||||
assert matches({"$eq": True}) == ["flag"]
|
||||
assert matches({"$in": ["classic", 2.5]}) == ["nums", "space"]
|
||||
assert matches({"$ne": "space"}) == ["flag", "nums", "strs"]
|
||||
assert matches({"$nin": [1, "2.5"]}) == ["flag", "scalar", "space"]
|
||||
|
||||
|
||||
def test_euclidean_distance(vectors, index):
|
||||
vectors.put_vectors(**index, vectors=[{"key": "p", "data": {"float32": [3, 4, 0, 0]}}])
|
||||
result = vectors.query_vectors(**index, queryVector={"float32": [0, 0, 0, 0]}, topK=1, returnDistance=True)
|
||||
assert result["distanceMetric"] == "euclidean"
|
||||
assert result["vectors"][0]["distance"] == pytest.approx(5.0)
|
||||
|
||||
|
||||
def test_cosine_metric(vectors):
|
||||
bucket = "cosine-" + uuid.uuid4().hex[:16]
|
||||
vectors.create_vector_bucket(vectorBucketName=bucket)
|
||||
@@ -270,17 +298,44 @@ def test_vector_policy_permissions(vectors, vector_clients, index):
|
||||
vectors.delete_index(**other)
|
||||
|
||||
|
||||
def index_object_encryption(vectors, bucket):
|
||||
s3 = boto3.client("s3", endpoint_url=vectors.meta.endpoint_url, aws_access_key_id="test",
|
||||
aws_secret_access_key="test", region_name="us-east-1")
|
||||
keys = [o["Key"] for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix=f"vector/{INDEX}/")
|
||||
for o in page.get("Contents", []) if o["Size"] > 0]
|
||||
assert any("/data/" in key for key in keys) and any("/_versions/" in key for key in keys)
|
||||
return {key: s3.head_object(Bucket=bucket, Key=key).get("ServerSideEncryption") for key in keys}
|
||||
|
||||
|
||||
def test_vector_bucket_encryption(vectors):
|
||||
bucket = "encryption-" + uuid.uuid4().hex[:16]
|
||||
plain = bucket + "-plain"
|
||||
config = {"sseType": "AES256"}
|
||||
vectors.create_vector_bucket(vectorBucketName=bucket, encryptionConfiguration=config)
|
||||
vectors.create_vector_bucket(vectorBucketName=plain)
|
||||
try:
|
||||
assert vectors.get_vector_bucket(vectorBucketName=bucket)["vectorBucket"]["encryptionConfiguration"] == config
|
||||
expect_error("ValidationException", vectors.create_vector_bucket, vectorBucketName=bucket + "-kms",
|
||||
encryptionConfiguration={"sseType": "aws:kms", "kmsKeyArn": "arn:aws:kms:us-east-1:123456789012:key/test"})
|
||||
assert vectors.list_vector_buckets(prefix=bucket + "-kms")["vectorBuckets"] == []
|
||||
|
||||
for name in (bucket, plain):
|
||||
vectors.create_index(vectorBucketName=name, indexName=INDEX, dataType="float32", dimension=4,
|
||||
distanceMetric="euclidean")
|
||||
vectors.put_vectors(vectorBucketName=name, indexName=INDEX, vectors=[
|
||||
{"key": "a", "data": {"float32": [1, 0, 0, 0]}, "metadata": {"color": "red"}},
|
||||
{"key": "b", "data": {"float32": [0, 1, 0, 0]}, "metadata": {"color": "blue"}},
|
||||
])
|
||||
hits = vectors.query_vectors(vectorBucketName=name, indexName=INDEX, queryVector={"float32": [1, 0, 0, 0]},
|
||||
topK=1, returnMetadata=True)["vectors"]
|
||||
assert [(h["key"], h["metadata"]) for h in hits] == [("a", {"color": "red"})]
|
||||
assert set(index_object_encryption(vectors, bucket).values()) == {"AES256"}
|
||||
assert set(index_object_encryption(vectors, plain).values()) == {None}
|
||||
finally:
|
||||
vectors.delete_vector_bucket(vectorBucketName=bucket)
|
||||
for name in (bucket, plain):
|
||||
with contextlib.suppress(ClientError):
|
||||
vectors.delete_index(vectorBucketName=name, indexName=INDEX)
|
||||
vectors.delete_vector_bucket(vectorBucketName=name)
|
||||
|
||||
|
||||
def test_catalog_pagination(vectors, index):
|
||||
@@ -362,6 +417,40 @@ def test_index_arn_validation(vectors, index):
|
||||
assert vectors.get_index(**index)["index"]["indexArn"] == arn
|
||||
|
||||
|
||||
def test_resource_tags(vectors, vector_clients, index):
|
||||
reader = vector_clients["reader"]
|
||||
bucket_arn = vectors.get_vector_bucket(vectorBucketName=index["vectorBucketName"])["vectorBucket"]["vectorBucketArn"]
|
||||
index_arn = vectors.get_index(**index)["index"]["indexArn"]
|
||||
assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {}
|
||||
vectors.tag_resource(resourceArn=bucket_arn, tags={"team": "search", "env": "dev"})
|
||||
vectors.tag_resource(resourceArn=index_arn, tags={"team": "ranking"})
|
||||
assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {"team": "search", "env": "dev"}
|
||||
assert vectors.list_tags_for_resource(resourceArn=index_arn)["tags"] == {"team": "ranking"}
|
||||
vectors.untag_resource(resourceArn=bucket_arn, tagKeys=["env", "absent"])
|
||||
assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {"team": "search"}
|
||||
expect_error("ValidationException", vectors.tag_resource, resourceArn=bucket_arn,
|
||||
tags={f"k{i}": "v" for i in range(50)})
|
||||
expect_error("NotFoundException", vectors.list_tags_for_resource, resourceArn=bucket_arn + "/index/missing")
|
||||
expect_error("AccessDeniedException", reader.list_tags_for_resource, resourceArn=bucket_arn)
|
||||
expect_error("AccessDeniedException", reader.tag_resource, resourceArn=index_arn, tags={"team": "other"})
|
||||
assert vectors.list_tags_for_resource(resourceArn=index_arn)["tags"] == {"team": "ranking"}
|
||||
|
||||
|
||||
def test_create_with_tags(vectors):
|
||||
bucket = "tagged-" + uuid.uuid4().hex[:16]
|
||||
bucket_arn = vectors.create_vector_bucket(vectorBucketName=bucket, tags={"team": "search"})["vectorBucketArn"]
|
||||
try:
|
||||
assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {"team": "search"}
|
||||
index_arn = vectors.create_index(vectorBucketName=bucket, indexName=INDEX, dataType="float32", dimension=4,
|
||||
distanceMetric="cosine", tags={"tier": "hot"})["indexArn"]
|
||||
try:
|
||||
assert vectors.list_tags_for_resource(resourceArn=index_arn)["tags"] == {"tier": "hot"}
|
||||
finally:
|
||||
vectors.delete_index(vectorBucketName=bucket, indexName=INDEX)
|
||||
finally:
|
||||
vectors.delete_vector_bucket(vectorBucketName=bucket)
|
||||
|
||||
|
||||
def test_invalid_credentials(cluster):
|
||||
client = boto3.client("s3vectors", endpoint_url=cluster["endpoint"], aws_access_key_id="unknown",
|
||||
aws_secret_access_key="unknown", region_name="us-east-1")
|
||||
@@ -389,3 +478,18 @@ def test_query_without_writer(vectors, index, cluster):
|
||||
raise
|
||||
time.sleep(0.2)
|
||||
assert [v["key"] for v in result["vectors"]] == ["survivor"]
|
||||
|
||||
|
||||
def test_engine_without_credentials(open_cluster):
|
||||
"""With no S3 identities, an engine given no credentials still serves an index."""
|
||||
client = boto3.client("s3vectors", endpoint_url=open_cluster["endpoint"], aws_access_key_id="any",
|
||||
aws_secret_access_key="any", region_name="us-east-1",
|
||||
config=Config(connect_timeout=3, read_timeout=20, retries={"max_attempts": 0}))
|
||||
index = {"vectorBucketName": BUCKET, "indexName": INDEX}
|
||||
client.create_vector_bucket(vectorBucketName=BUCKET)
|
||||
client.create_index(**index, dataType="float32", dimension=2, distanceMetric="cosine")
|
||||
client.put_vectors(**index, vectors=[{"key": "a", "data": {"float32": [1, 0]}},
|
||||
{"key": "b", "data": {"float32": [0, 1]}}])
|
||||
hits = client.query_vectors(**index, queryVector={"float32": [1, 0]}, topK=1)["vectors"]
|
||||
assert [h["key"] for h in hits] == ["a"]
|
||||
|
||||
@@ -43,6 +43,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/scheduler"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
|
||||
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/credential/grpc" // Register gRPC credential store
|
||||
@@ -167,14 +168,21 @@ type AdminServer struct {
|
||||
collectionStatsCacheThreshold time.Duration
|
||||
|
||||
s3TablesManager *s3tables.Manager
|
||||
s3VectorsServer *s3vectors.Server
|
||||
icebergPort int
|
||||
lancePort int
|
||||
|
||||
// s3PublicEndpoint is the client-facing S3 address from admin.toml
|
||||
// (s3.public_endpoint); discovered S3 servers are the fallback.
|
||||
s3PublicEndpoint string
|
||||
|
||||
// uploadMaxSize caps the total bytes of one file browser upload; 0 means no limit.
|
||||
uploadMaxSize int64
|
||||
}
|
||||
|
||||
// DefaultUploadMaxSizeMB is the default total size limit of one file browser upload.
|
||||
const DefaultUploadMaxSizeMB = 500
|
||||
|
||||
// Type definitions moved to types.go
|
||||
|
||||
func NewAdminServer(masters string, filerGroup string, templateFS http.FileSystem, dataDir string, icebergPort, lancePort int, s3PublicEndpoint string) *AdminServer {
|
||||
@@ -215,11 +223,14 @@ func NewAdminServer(masters string, filerGroup string, templateFS http.FileSyste
|
||||
icebergPort: icebergPort,
|
||||
lancePort: lancePort,
|
||||
s3PublicEndpoint: normalizeS3PublicEndpoint(s3PublicEndpoint),
|
||||
uploadMaxSize: DefaultUploadMaxSizeMB << 20,
|
||||
pluginLock: lockManager,
|
||||
adminPresenceLock: presenceLock,
|
||||
bgCancel: bgCancel,
|
||||
}
|
||||
|
||||
server.s3VectorsServer = newS3VectorsServer(server)
|
||||
|
||||
// Initialize topic retention purger
|
||||
server.topicRetentionPurger = NewTopicRetentionPurger(server)
|
||||
|
||||
@@ -1570,6 +1581,18 @@ func (s *AdminServer) GetS3Endpoint() string {
|
||||
return s.s3PublicEndpoint
|
||||
}
|
||||
|
||||
// SetUploadMaxSize sets the total size limit in bytes of one file browser
|
||||
// upload; zero or less removes the limit.
|
||||
func (s *AdminServer) SetUploadMaxSize(maxBytes int64) {
|
||||
s.uploadMaxSize = max(maxBytes, 0)
|
||||
}
|
||||
|
||||
// GetUploadMaxSize returns the total size limit in bytes of one file browser
|
||||
// upload, or 0 when uploads are unlimited.
|
||||
func (s *AdminServer) GetUploadMaxSize() int64 {
|
||||
return s.uploadMaxSize
|
||||
}
|
||||
|
||||
// normalizeS3PublicEndpoint trims a trailing slash and drops, with a warning,
|
||||
// a value that is not a plain absolute http or https URL, so the file browser
|
||||
// hides its URL actions instead of copying broken links. Checking the string
|
||||
|
||||
@@ -53,6 +53,8 @@ type FileBrowserData struct {
|
||||
HasNextPage bool `json:"has_next_page"`
|
||||
LastFileName string `json:"last_file_name"` // Cursor for next page
|
||||
CurrentLastFileName string `json:"current_last_file_name"` // Cursor from current request (for page size changes)
|
||||
|
||||
UploadMaxSize int64 `json:"upload_max_size"`
|
||||
}
|
||||
|
||||
// GetFileBrowser retrieves file browser data for a given path with cursor-based
|
||||
@@ -240,6 +242,8 @@ func (s *AdminServer) GetFileBrowser(dir string, prefix string, lastFileName str
|
||||
HasNextPage: hasNextPage,
|
||||
LastFileName: lastEntryName, // Store for next page navigation
|
||||
CurrentLastFileName: lastFileName, // Store input cursor for page size changes
|
||||
|
||||
UploadMaxSize: s.uploadMaxSize,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
|
||||
)
|
||||
|
||||
type S3VectorsBucketsData struct {
|
||||
Username string `json:"username"`
|
||||
Buckets []S3VectorsBucketSummary `json:"buckets"`
|
||||
TotalBuckets int `json:"total_buckets"`
|
||||
Truncated bool `json:"truncated,omitempty"`
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
}
|
||||
|
||||
type S3VectorsBucketSummary struct {
|
||||
Name string `json:"name"`
|
||||
ARN string `json:"arn"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
// Encryption is the bucket's sseType, or empty when it declares none.
|
||||
Encryption string `json:"encryption,omitempty"`
|
||||
}
|
||||
|
||||
type S3VectorsBucketData struct {
|
||||
Username string `json:"username"`
|
||||
Bucket S3VectorsBucketSummary `json:"bucket"`
|
||||
Policy string `json:"policy,omitempty"`
|
||||
Indexes []*s3vectors.Index `json:"indexes"`
|
||||
TotalIndexes int `json:"total_indexes"`
|
||||
Truncated bool `json:"truncated,omitempty"`
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
}
|
||||
|
||||
// The S3 Vectors operations reach the filer through the admin server's own
|
||||
// filer connection.
|
||||
type s3VectorsFilerClient struct{ s *AdminServer }
|
||||
|
||||
func (c s3VectorsFilerClient) WithFilerClient(_ bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
|
||||
return c.s.WithFilerClient(fn)
|
||||
}
|
||||
|
||||
// newS3VectorsServer serves the catalog side of S3 Vectors for the admin
|
||||
// console. It has no vector engine, so it never runs an operation that needs
|
||||
// one, such as CreateIndex.
|
||||
func newS3VectorsServer(s *AdminServer) *s3vectors.Server {
|
||||
server := s3vectors.NewServer(s3VectorsFilerClient{s}, nil, nil)
|
||||
server.SetRegion(s3tables.DefaultRegion)
|
||||
server.SetAccountID(s3tables.DefaultAccountID)
|
||||
return server
|
||||
}
|
||||
|
||||
func (s *AdminServer) executeS3VectorsOperation(ctx context.Context, operation string, req, resp interface{}) error {
|
||||
return s.s3VectorsServer.Execute(ctx, operation, req, resp, s3_constants.AccountAdminId)
|
||||
}
|
||||
|
||||
const s3VectorsAdminPage = 500
|
||||
|
||||
func (s *AdminServer) s3VectorsBucketSummary(ctx context.Context, name string) (S3VectorsBucketSummary, error) {
|
||||
var resp s3vectors.GetVectorBucketResponse
|
||||
if err := s.executeS3VectorsOperation(ctx, "GetVectorBucket", &s3vectors.GetVectorBucketRequest{VectorBucketName: name}, &resp); err != nil {
|
||||
return S3VectorsBucketSummary{}, err
|
||||
}
|
||||
b := resp.VectorBucket
|
||||
summary := S3VectorsBucketSummary{Name: b.VectorBucketName, ARN: b.VectorBucketArn, CreatedAt: b.CreationTime}
|
||||
if sse, ok := b.EncryptionConfiguration["sseType"].(string); ok {
|
||||
summary.Encryption = sse
|
||||
}
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) GetS3VectorsBucketsData(ctx context.Context) (S3VectorsBucketsData, error) {
|
||||
data := S3VectorsBucketsData{Buckets: []S3VectorsBucketSummary{}}
|
||||
page := s3VectorsAdminPage
|
||||
req := &s3vectors.ListVectorBucketsRequest{MaxResults: &page}
|
||||
for {
|
||||
var resp s3vectors.ListVectorBucketsResponse
|
||||
if err := s.executeS3VectorsOperation(ctx, "ListVectorBuckets", req, &resp); err != nil {
|
||||
return S3VectorsBucketsData{}, err
|
||||
}
|
||||
for _, b := range resp.VectorBuckets {
|
||||
if len(data.Buckets) == s3TablesAdminListLimit {
|
||||
data.Truncated = true
|
||||
break
|
||||
}
|
||||
summary, err := s.s3VectorsBucketSummary(ctx, b.VectorBucketName)
|
||||
if isS3VectorsNotFound(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return S3VectorsBucketsData{}, err
|
||||
}
|
||||
data.Buckets = append(data.Buckets, summary)
|
||||
}
|
||||
if data.Truncated || resp.NextToken == "" {
|
||||
break
|
||||
}
|
||||
req.NextToken = resp.NextToken
|
||||
}
|
||||
data.TotalBuckets = len(data.Buckets)
|
||||
data.LastUpdated = time.Now()
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) GetS3VectorsBucketData(ctx context.Context, bucket string) (S3VectorsBucketData, error) {
|
||||
summary, err := s.s3VectorsBucketSummary(ctx, bucket)
|
||||
if err != nil {
|
||||
return S3VectorsBucketData{}, err
|
||||
}
|
||||
data := S3VectorsBucketData{Bucket: summary, Indexes: []*s3vectors.Index{}, LastUpdated: time.Now()}
|
||||
var policy s3vectors.GetVectorBucketPolicyResponse
|
||||
switch err := s.executeS3VectorsOperation(ctx, "GetVectorBucketPolicy", &s3vectors.GetVectorBucketPolicyRequest{VectorBucketName: bucket}, &policy); {
|
||||
case err == nil:
|
||||
data.Policy = policy.Policy
|
||||
case !isS3VectorsNotFound(err):
|
||||
return S3VectorsBucketData{}, err
|
||||
}
|
||||
page := s3VectorsAdminPage
|
||||
req := &s3vectors.ListIndexesRequest{VectorBucketName: bucket, MaxResults: &page}
|
||||
for {
|
||||
var resp s3vectors.ListIndexesResponse
|
||||
if err := s.executeS3VectorsOperation(ctx, "ListIndexes", req, &resp); err != nil {
|
||||
return S3VectorsBucketData{}, err
|
||||
}
|
||||
for _, summary := range resp.Indexes {
|
||||
if len(data.Indexes) == s3TablesAdminListLimit {
|
||||
data.Truncated = true
|
||||
break
|
||||
}
|
||||
var index s3vectors.GetIndexResponse
|
||||
err := s.executeS3VectorsOperation(ctx, "GetIndex", &s3vectors.GetIndexRequest{VectorBucketName: bucket, IndexName: summary.IndexName}, &index)
|
||||
if isS3VectorsNotFound(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return S3VectorsBucketData{}, err
|
||||
}
|
||||
data.Indexes = append(data.Indexes, index.Index)
|
||||
}
|
||||
if data.Truncated || resp.NextToken == "" {
|
||||
break
|
||||
}
|
||||
req.NextToken = resp.NextToken
|
||||
}
|
||||
data.TotalIndexes = len(data.Indexes)
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) ListS3VectorsBucketsAPI(w http.ResponseWriter, r *http.Request) {
|
||||
data, err := s.GetS3VectorsBucketsData(r.Context())
|
||||
if err != nil {
|
||||
writeS3VectorsError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, data)
|
||||
}
|
||||
|
||||
func (s *AdminServer) CreateS3VectorsBucket(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
var resp s3vectors.CreateVectorBucketResponse
|
||||
if err := s.executeS3VectorsOperation(r.Context(), "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: req.Name}, &resp); err != nil {
|
||||
writeS3VectorsError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, map[string]interface{}{"arn": resp.VectorBucketArn})
|
||||
}
|
||||
|
||||
func (s *AdminServer) DeleteS3VectorsBucket(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
bucket := r.URL.Query().Get("bucket")
|
||||
if bucket == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket query parameter is required")
|
||||
return
|
||||
}
|
||||
if err := s.executeS3VectorsOperation(r.Context(), "DeleteVectorBucket", &s3vectors.DeleteVectorBucketRequest{VectorBucketName: bucket}, nil); err != nil {
|
||||
writeS3VectorsError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Vector bucket deleted"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) GetS3VectorsBucketAPI(w http.ResponseWriter, r *http.Request) {
|
||||
bucket := r.URL.Query().Get("bucket")
|
||||
if bucket == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket query parameter is required")
|
||||
return
|
||||
}
|
||||
data, err := s.GetS3VectorsBucketData(r.Context(), bucket)
|
||||
if err != nil {
|
||||
writeS3VectorsError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, data)
|
||||
}
|
||||
|
||||
func (s *AdminServer) DeleteS3VectorsIndex(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireSessionCSRFToken(w, r) {
|
||||
return
|
||||
}
|
||||
bucket, index := r.URL.Query().Get("bucket"), r.URL.Query().Get("index")
|
||||
if bucket == "" || index == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "bucket and index query parameters are required")
|
||||
return
|
||||
}
|
||||
if err := s.executeS3VectorsOperation(r.Context(), "DeleteIndex", &s3vectors.DeleteIndexRequest{VectorBucketName: bucket, IndexName: index}, nil); err != nil {
|
||||
writeS3VectorsError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Index deleted"})
|
||||
}
|
||||
|
||||
func isS3VectorsNotFound(err error) bool {
|
||||
var e *s3vectors.Error
|
||||
return errors.As(err, &e) && e.Code == "NotFoundException"
|
||||
}
|
||||
|
||||
func writeS3VectorsError(w http.ResponseWriter, err error) {
|
||||
var e *s3vectors.Error
|
||||
if !errors.As(err, &e) {
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
status := http.StatusInternalServerError
|
||||
switch e.Code {
|
||||
case "ValidationException":
|
||||
status = http.StatusBadRequest
|
||||
case "AccessDeniedException":
|
||||
status = http.StatusForbidden
|
||||
case "NotFoundException":
|
||||
status = http.StatusNotFound
|
||||
case "ConflictException":
|
||||
status = http.StatusConflict
|
||||
case "ServiceUnavailableException":
|
||||
status = http.StatusServiceUnavailable
|
||||
}
|
||||
writeJSONError(w, status, e.Message)
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
|
||||
)
|
||||
|
||||
func newS3VectorsTestServer(t *testing.T) (*AdminServer, *s3tables.ManagerClient) {
|
||||
t.Helper()
|
||||
fs := s3tablestest.Start(t)
|
||||
client := s3tables.NewManagerClient(fs.Client)
|
||||
server := s3vectors.NewServer(client, nil, nil)
|
||||
server.SetRegion(s3tables.DefaultRegion)
|
||||
server.SetAccountID(s3tables.DefaultAccountID)
|
||||
return &AdminServer{s3VectorsServer: server}, client
|
||||
}
|
||||
|
||||
// createS3VectorsIndexRecord writes the catalog record CreateIndex leaves
|
||||
// behind; the admin server has no engine to run CreateIndex itself.
|
||||
func createS3VectorsIndexRecord(t *testing.T, client *s3tables.ManagerClient, bucket, index string, metadata map[string]string) {
|
||||
t.Helper()
|
||||
m := s3tables.NewManager()
|
||||
m.SetTrusted(true)
|
||||
arn := "arn:aws:s3tables:us-east-1:000000000000:bucket/" + bucket
|
||||
ctx := context.Background()
|
||||
err := m.Execute(ctx, client, "CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}}, nil, s3_constants.AccountAdminId)
|
||||
if err != nil && !strings.Contains(err.Error(), "already exists") {
|
||||
t.Fatalf("create namespace: %v", err)
|
||||
}
|
||||
full, _ := json.Marshal(metadata)
|
||||
if err := m.Execute(ctx, client, "CreateTable", &s3tables.CreateTableRequest{
|
||||
TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}, Name: index,
|
||||
Format: s3tables.FormatLance, Metadata: &s3tables.TableMetadata{FullMetadata: full},
|
||||
}, nil, s3_constants.AccountAdminId); err != nil {
|
||||
t.Fatalf("create index record: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func s3VectorsRequest(t *testing.T, handler http.HandlerFunc, method, target, body string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
handler(w, httptest.NewRequest(method, target, strings.NewReader(body)))
|
||||
return w
|
||||
}
|
||||
|
||||
func TestS3VectorsBucketsListsOnlyVectorBuckets(t *testing.T) {
|
||||
s, client := newS3VectorsTestServer(t)
|
||||
ctx := context.Background()
|
||||
if w := s3VectorsRequest(t, s.CreateS3VectorsBucket, http.MethodPost, "/api/s3vectors/buckets", `{"name":"media"}`); w.Code != http.StatusCreated {
|
||||
t.Fatalf("create: %d %s", w.Code, w.Body)
|
||||
}
|
||||
if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{
|
||||
VectorBucketName: "sealed", EncryptionConfiguration: map[string]interface{}{"sseType": "AES256"},
|
||||
}, nil); err != nil {
|
||||
t.Fatalf("create sealed: %v", err)
|
||||
}
|
||||
if err := s3tables.NewManager().Execute(ctx, client, "CreateTableBucket", &s3tables.CreateTableBucketRequest{
|
||||
Name: "warehouse", Format: s3tables.FormatIceberg,
|
||||
}, nil, s3_constants.AccountAdminId); err != nil {
|
||||
t.Fatalf("create iceberg bucket: %v", err)
|
||||
}
|
||||
|
||||
data, err := s.GetS3VectorsBucketsData(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := map[string]string{}
|
||||
for _, b := range data.Buckets {
|
||||
got[b.Name] = b.Encryption
|
||||
if want := "arn:aws:s3vectors:us-east-1:000000000000:bucket/" + b.Name; b.ARN != want || b.CreatedAt.IsZero() {
|
||||
t.Errorf("%s: arn %q created %v, want %q and a time", b.Name, b.ARN, b.CreatedAt, want)
|
||||
}
|
||||
}
|
||||
if len(got) != 2 || got["media"] != "" || got["sealed"] != "AES256" || data.TotalBuckets != 2 || data.Truncated {
|
||||
t.Fatalf("buckets %v (total %d), want media and sealed (AES256) without the iceberg bucket", got, data.TotalBuckets)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3VectorsBucketShowsIndexesAndPolicy(t *testing.T) {
|
||||
s, client := newS3VectorsTestServer(t)
|
||||
ctx := context.Background()
|
||||
if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: "media"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
createS3VectorsIndexRecord(t, client, "media", "movies", map[string]string{
|
||||
"dimension": "3", "distanceMetric": "cosine", "nonFilterableMetadataKeys": `["summary"]`,
|
||||
})
|
||||
policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3vectors:QueryVectors","Resource":"*"}]}`
|
||||
if err := s.executeS3VectorsOperation(ctx, "PutVectorBucketPolicy", &s3vectors.PutVectorBucketPolicyRequest{VectorBucketName: "media", Policy: policy}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
w := s3VectorsRequest(t, s.GetS3VectorsBucketAPI, http.MethodGet, "/api/s3vectors/indexes?bucket=media", "")
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("get: %d %s", w.Code, w.Body)
|
||||
}
|
||||
var data S3VectorsBucketData
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if data.Policy != policy {
|
||||
t.Errorf("policy %q, want %q", data.Policy, policy)
|
||||
}
|
||||
if data.TotalIndexes != 1 {
|
||||
t.Fatalf("indexes %+v", data.Indexes)
|
||||
}
|
||||
idx := data.Indexes[0]
|
||||
if idx.IndexName != "movies" || idx.Dimension != 3 || idx.DistanceMetric != "cosine" || idx.DataType != "float32" ||
|
||||
idx.MetadataConfiguration == nil || len(idx.MetadataConfiguration.NonFilterableMetadataKeys) != 1 {
|
||||
t.Fatalf("index %+v", idx)
|
||||
}
|
||||
|
||||
if w := s3VectorsRequest(t, s.GetS3VectorsBucketAPI, http.MethodGet, "/api/s3vectors/indexes?bucket=missing", ""); w.Code != http.StatusNotFound {
|
||||
t.Fatalf("missing bucket: %d, want 404", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3VectorsDeleteIndexThenBucket(t *testing.T) {
|
||||
s, client := newS3VectorsTestServer(t)
|
||||
ctx := context.Background()
|
||||
if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: "media"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
createS3VectorsIndexRecord(t, client, "media", "movies", map[string]string{"dimension": "3", "distanceMetric": "cosine"})
|
||||
|
||||
if w := s3VectorsRequest(t, s.DeleteS3VectorsBucket, http.MethodDelete, "/api/s3vectors/buckets?bucket=media", ""); w.Code != http.StatusConflict {
|
||||
t.Fatalf("delete a bucket with an index: %d %s, want 409", w.Code, w.Body)
|
||||
}
|
||||
if w := s3VectorsRequest(t, s.DeleteS3VectorsIndex, http.MethodDelete, "/api/s3vectors/indexes?bucket=media&index=movies", ""); w.Code != http.StatusOK {
|
||||
t.Fatalf("delete index: %d %s", w.Code, w.Body)
|
||||
}
|
||||
// The emptied index namespace does not keep the bucket alive.
|
||||
if w := s3VectorsRequest(t, s.DeleteS3VectorsBucket, http.MethodDelete, "/api/s3vectors/buckets?bucket=media", ""); w.Code != http.StatusOK {
|
||||
t.Fatalf("delete bucket: %d %s", w.Code, w.Body)
|
||||
}
|
||||
data, err := s.GetS3VectorsBucketsData(ctx)
|
||||
if err != nil || data.TotalBuckets != 0 {
|
||||
t.Fatalf("after delete: %+v, %v", data.Buckets, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3VectorsBucketsStopAtTheListLimit(t *testing.T) {
|
||||
s, client := newS3VectorsTestServer(t)
|
||||
ctx := context.Background()
|
||||
// The ICEBERG bucket leaves the first page short, so the last page
|
||||
// would carry the list past the limit.
|
||||
if err := s3tables.NewManager().Execute(ctx, client, "CreateTableBucket", &s3tables.CreateTableBucketRequest{
|
||||
Name: "iceberg", Format: s3tables.FormatIceberg,
|
||||
}, nil, s3_constants.AccountAdminId); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i <= s3TablesAdminListLimit; i++ {
|
||||
if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: fmt.Sprintf("media-%04d", i)}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
data, err := s.GetS3VectorsBucketsData(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if data.TotalBuckets != s3TablesAdminListLimit || !data.Truncated {
|
||||
t.Fatalf("listed %d buckets (truncated %v), want the limit %d and truncated", data.TotalBuckets, data.Truncated, s3TablesAdminListLimit)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
)
|
||||
|
||||
@@ -111,6 +113,8 @@ func (h *AdminHandlers) registerUIRoutes(r *mux.Router) {
|
||||
r.HandleFunc("/object-store/groups", h.groupHandlers.ShowGroups).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/service-accounts", h.serviceAccountHandlers.ShowServiceAccounts).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/s3tables/buckets", h.ShowS3TablesBuckets).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/vector-buckets", h.ShowS3VectorsBuckets).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/vector-buckets/{bucket}", h.ShowS3VectorsBucketDetails).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces", h.ShowS3TablesNamespaces).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces/{namespace}/tables", h.ShowS3TablesTables).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces/{namespace}/tables/{table}", h.ShowS3TablesTableDetails).Methods(http.MethodGet)
|
||||
@@ -251,6 +255,13 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
|
||||
s3TablesApi.HandleFunc("/tags", h.adminServer.ListS3TablesTags).Methods(http.MethodGet)
|
||||
s3TablesApi.Handle("/tags", wrapWrite(h.adminServer.UntagS3TablesResource)).Methods(http.MethodDelete)
|
||||
|
||||
s3VectorsApi := api.PathPrefix("/s3vectors").Subrouter()
|
||||
s3VectorsApi.HandleFunc("/buckets", h.adminServer.ListS3VectorsBucketsAPI).Methods(http.MethodGet)
|
||||
s3VectorsApi.Handle("/buckets", wrapWrite(h.adminServer.CreateS3VectorsBucket)).Methods(http.MethodPost)
|
||||
s3VectorsApi.Handle("/buckets", wrapWrite(h.adminServer.DeleteS3VectorsBucket)).Methods(http.MethodDelete)
|
||||
s3VectorsApi.HandleFunc("/indexes", h.adminServer.GetS3VectorsBucketAPI).Methods(http.MethodGet)
|
||||
s3VectorsApi.Handle("/indexes", wrapWrite(h.adminServer.DeleteS3VectorsIndex)).Methods(http.MethodDelete)
|
||||
|
||||
filesApi := api.PathPrefix("/files").Subrouter()
|
||||
filesApi.Handle("/delete", wrapWrite(h.fileBrowserHandlers.DeleteFile)).Methods(http.MethodDelete)
|
||||
filesApi.Handle("/delete-multiple", wrapWrite(h.fileBrowserHandlers.DeleteMultipleFiles)).Methods(http.MethodDelete)
|
||||
@@ -373,6 +384,45 @@ func (h *AdminHandlers) ShowS3TablesBuckets(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
}
|
||||
|
||||
// ShowS3VectorsBuckets renders the vector buckets page
|
||||
func (h *AdminHandlers) ShowS3VectorsBuckets(w http.ResponseWriter, r *http.Request) {
|
||||
username := h.getUsername(r)
|
||||
data, err := h.adminServer.GetS3VectorsBucketsData(r.Context())
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get vector buckets: "+err.Error())
|
||||
return
|
||||
}
|
||||
data.Username = username
|
||||
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
if err := layout.Layout(viewCtx, app.S3VectorsBuckets(data)).Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// ShowS3VectorsBucketDetails renders one vector bucket with its indexes and policy
|
||||
func (h *AdminHandlers) ShowS3VectorsBucketDetails(w http.ResponseWriter, r *http.Request) {
|
||||
username := h.getUsername(r)
|
||||
data, err := h.adminServer.GetS3VectorsBucketData(r.Context(), mux.Vars(r)["bucket"])
|
||||
if err != nil {
|
||||
status := http.StatusInternalServerError
|
||||
var e *s3vectors.Error
|
||||
if errors.As(err, &e) && e.Code == "NotFoundException" {
|
||||
status = http.StatusNotFound
|
||||
}
|
||||
writeJSONError(w, status, "Failed to get vector bucket: "+err.Error())
|
||||
return
|
||||
}
|
||||
data.Username = username
|
||||
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
|
||||
if err := layout.Layout(viewCtx, app.S3VectorsBucketDetails(data)).Render(r.Context(), w); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// ShowS3TablesNamespaces renders namespaces for a table bucket
|
||||
func (h *AdminHandlers) ShowS3TablesNamespaces(w http.ResponseWriter, r *http.Request) {
|
||||
username := h.getUsername(r)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bufio"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
@@ -247,25 +248,46 @@ func (h *FileBrowserHandlers) CreateFolder(w http.ResponseWriter, r *http.Reques
|
||||
|
||||
// UploadFile handles file upload requests
|
||||
func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request) {
|
||||
// Get the current path
|
||||
maxSize := h.adminServer.GetUploadMaxSize()
|
||||
if maxSize > 0 {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxSize+uploadFormOverhead)
|
||||
}
|
||||
|
||||
// Parts beyond 32MB spill to temporary files instead of memory
|
||||
err := r.ParseMultipartForm(32 << 20)
|
||||
if err != nil {
|
||||
var maxBytesErr *http.MaxBytesError
|
||||
if errors.As(err, &maxBytesErr) {
|
||||
writeJSONError(w, http.StatusRequestEntityTooLarge, uploadTooLargeMessage(maxSize))
|
||||
return
|
||||
}
|
||||
writeJSONError(w, http.StatusBadRequest, "Failed to parse multipart form: "+err.Error())
|
||||
return
|
||||
}
|
||||
// The router hands us a copy of the request, so the server cannot remove the spooled parts itself
|
||||
defer r.MultipartForm.RemoveAll()
|
||||
|
||||
currentPath := r.FormValue("path")
|
||||
if currentPath == "" {
|
||||
currentPath = "/"
|
||||
}
|
||||
|
||||
// Parse multipart form
|
||||
err := r.ParseMultipartForm(1 << 30) // 1GB max memory for large file uploads
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Failed to parse multipart form: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Get uploaded files (supports multiple files)
|
||||
files := r.MultipartForm.File["files"]
|
||||
if len(files) == 0 {
|
||||
writeJSONError(w, http.StatusBadRequest, "No files uploaded")
|
||||
return
|
||||
}
|
||||
if maxSize > 0 {
|
||||
var totalSize int64
|
||||
for _, fileHeader := range files {
|
||||
totalSize += fileHeader.Size
|
||||
}
|
||||
if totalSize > maxSize {
|
||||
writeJSONError(w, http.StatusRequestEntityTooLarge, uploadTooLargeMessage(maxSize))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
var uploadResults []map[string]interface{}
|
||||
var failedUploads []string
|
||||
@@ -327,6 +349,15 @@ func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
|
||||
// uploadFormOverhead allows for the multipart headers and the path field on
|
||||
// top of the file bytes an upload may carry. Go's multipart parser accepts at
|
||||
// most 1000 parts, whose headers take about half of this.
|
||||
const uploadFormOverhead = 1 << 20
|
||||
|
||||
func uploadTooLargeMessage(maxSize int64) string {
|
||||
return fmt.Sprintf("Total upload size exceeds the %d MB limit", maxSize>>20)
|
||||
}
|
||||
|
||||
// uploadFileToFiler uploads a file to the cluster via filer gRPC + volume
|
||||
// HTTP. This works whether or not the filer is running with -disableHttp=true,
|
||||
// since the bytes never traverse the filer's HTTP listener. The multipart
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -143,3 +147,72 @@ func TestListFolders_RejectsPathOutsideBuckets(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadFile_RejectsOversizedUpload(t *testing.T) {
|
||||
adminServer := &dash.AdminServer{}
|
||||
adminServer.SetUploadMaxSize(1 << 20)
|
||||
h := &FileBrowserHandlers{adminServer: adminServer}
|
||||
|
||||
// 1.5MB and two 600KB files fit the body allowance but exceed the file total;
|
||||
// 3MB trips the body cap.
|
||||
for _, sizes := range [][]int{{3 << 19}, {600 << 10, 600 << 10}, {3 << 20}} {
|
||||
var body bytes.Buffer
|
||||
mw := multipart.NewWriter(&body)
|
||||
for i, size := range sizes {
|
||||
part, err := mw.CreateFormFile("files", fmt.Sprintf("big%d.bin", i))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
part.Write(make([]byte, size))
|
||||
}
|
||||
mw.WriteField("path", "/buckets/b")
|
||||
mw.Close()
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/files/upload", &body)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
h.UploadFile(w, req)
|
||||
|
||||
if w.Code != http.StatusRequestEntityTooLarge {
|
||||
t.Errorf("sizes %v: expected 413, got %d (body: %s)", sizes, w.Code, w.Body.String())
|
||||
}
|
||||
if !strings.Contains(w.Body.String(), "1 MB limit") {
|
||||
t.Errorf("sizes %v: expected the limit in the error, got %s", sizes, w.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadFile_RemovesSpooledParts(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
t.Setenv("TMPDIR", tmpDir)
|
||||
|
||||
adminServer := &dash.AdminServer{}
|
||||
adminServer.SetUploadMaxSize(34 << 20)
|
||||
h := &FileBrowserHandlers{adminServer: adminServer}
|
||||
|
||||
// The 33MB part spills to disk; together the parts exceed the limit, so the upload stops there.
|
||||
var body bytes.Buffer
|
||||
mw := multipart.NewWriter(&body)
|
||||
for i, size := range []int{33 << 20, 3 << 19} {
|
||||
part, err := mw.CreateFormFile("files", fmt.Sprintf("part%d.bin", i))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
part.Write(make([]byte, size))
|
||||
}
|
||||
mw.Close()
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/files/upload", &body)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
h.UploadFile(w, req.WithContext(req.Context()))
|
||||
|
||||
if w.Code != http.StatusRequestEntityTooLarge {
|
||||
t.Fatalf("expected 413, got %d (body: %s)", w.Code, w.Body.String())
|
||||
}
|
||||
if left, _ := os.ReadDir(tmpDir); len(left) != 0 {
|
||||
t.Errorf("spooled parts left behind: %v", left)
|
||||
}
|
||||
}
|
||||
@@ -1180,10 +1180,9 @@ async function submitUploadFile() {
|
||||
const files = Array.from(fileInput.files);
|
||||
const totalSize = files.reduce((sum, file) => sum + file.size, 0);
|
||||
|
||||
// Validate total file size (limit to 500MB for admin interface)
|
||||
const maxSize = 500 * 1024 * 1024; // 500MB total
|
||||
if (totalSize > maxSize) {
|
||||
showErrorMessage('Total file size exceeds 500MB limit. Please select fewer or smaller files.');
|
||||
const maxSize = Number(document.getElementById('uploadFileForm').dataset.maxSize) || 0;
|
||||
if (maxSize > 0 && totalSize > maxSize) {
|
||||
showErrorMessage(`Total file size exceeds the ${formatBytes(maxSize)} limit. Please select fewer or smaller files.`);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
Binary file not shown.
@@ -310,12 +310,15 @@ templ FileBrowser(data dash.FileBrowserData) {
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<form id="uploadFileForm" enctype="multipart/form-data">
|
||||
<form id="uploadFileForm" enctype="multipart/form-data" data-max-size={ fmt.Sprint(data.UploadMaxSize) }>
|
||||
<div class="mb-3">
|
||||
<label for="fileInput" class="form-label">Select Files</label>
|
||||
<input type="file" class="form-control" id="fileInput" name="files" multiple required>
|
||||
<div class="form-text">
|
||||
Choose one or more files to upload to the current directory. You can select multiple files by holding Ctrl (Cmd on Mac) while clicking.
|
||||
if data.UploadMaxSize > 0 {
|
||||
Total upload size limit: { formatBytes(data.UploadMaxSize) }.
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
<input type="hidden" id="uploadPath" name="path" value={ data.CurrentPath }>
|
||||
|
||||
@@ -852,20 +852,56 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 93, "\"></form></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-primary\" onclick=\"submitCreateFolder()\"><i class=\"bi bi-folder-plus me-1\"></i>Create Folder</button></div></div></div></div><!-- Upload File Modal --><div class=\"modal fade\" id=\"uploadFileModal\" tabindex=\"-1\" aria-labelledby=\"uploadFileModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog modal-lg\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"uploadFileModalLabel\"><i class=\"bi bi-upload me-2\"></i>Upload Files</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><form id=\"uploadFileForm\" enctype=\"multipart/form-data\"><div class=\"mb-3\"><label for=\"fileInput\" class=\"form-label\">Select Files</label> <input type=\"file\" class=\"form-control\" id=\"fileInput\" name=\"files\" multiple required><div class=\"form-text\">Choose one or more files to upload to the current directory. You can select multiple files by holding Ctrl (Cmd on Mac) while clicking.</div></div><input type=\"hidden\" id=\"uploadPath\" name=\"path\" value=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 93, "\"></form></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-primary\" onclick=\"submitCreateFolder()\"><i class=\"bi bi-folder-plus me-1\"></i>Create Folder</button></div></div></div></div><!-- Upload File Modal --><div class=\"modal fade\" id=\"uploadFileModal\" tabindex=\"-1\" aria-labelledby=\"uploadFileModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog modal-lg\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"uploadFileModalLabel\"><i class=\"bi bi-upload me-2\"></i>Upload Files</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><form id=\"uploadFileForm\" enctype=\"multipart/form-data\" data-max-size=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var40 string
|
||||
templ_7745c5c3_Var40, templ_7745c5c3_Err = templruntime.NewAttributeExpression(data.CurrentPath).Resolve(ctx)
|
||||
templ_7745c5c3_Var40, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprint(data.UploadMaxSize)).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 321, Col: 79}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 313, Col: 107}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var40)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, "\"><!-- File List Preview --><div id=\"fileListPreview\" class=\"mb-3\" style=\"display: none;\"><label class=\"form-label\">Selected Files:</label><div id=\"selectedFilesList\" class=\"border rounded p-2 bg-light\"><!-- Files will be listed here --></div></div><!-- Upload Progress --><div class=\"mb-3\" id=\"uploadProgress\" style=\"display: none;\"><label class=\"form-label\">Upload Progress:</label><div class=\"progress mb-2\"><div class=\"progress-bar progress-bar-striped progress-bar-animated\" role=\"progressbar\" style=\"width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\">0%</div></div><div id=\"uploadStatus\" class=\"small text-muted\">Preparing upload...</div></div></form></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-primary\" onclick=\"submitUploadFile()\"><i class=\"bi bi-upload me-1\"></i>Upload Files</button></div></div></div></div><!-- JavaScript for file browser functionality --><script>\n\tdocument.addEventListener('DOMContentLoaded', function() {\n\t\t// Format permissions in the main table\n\t\tdocument.querySelectorAll('.permissions-display').forEach(element => {\n\t\t\tconst mode = element.getAttribute('data-mode');\n\t\t\tconst isDirectory = element.getAttribute('data-is-directory') === 'true';\n\t\t\tif (mode) {\n\t\t\t\telement.textContent = formatPermissions(mode, isDirectory);\n\t\t\t}\n\t\t});\n\t\t\n\t\t// Handle file browser action buttons (download, view, properties, export-metadata, delete)\n\t\tdocument.addEventListener('click', function(e) {\n\t\t\tconst button = e.target.closest('[data-action]');\n\t\t\tif (!button) return;\n\t\t\t\n\t\t\tconst action = button.getAttribute('data-action');\n\t\t\tif (action === 'copy-url') {\n\t\t\t\tconst url = button.getAttribute('data-url');\n\t\t\t\tif (url) adminCopyToClipboard(url);\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tconst path = button.getAttribute('data-path');\n\n\t\t\tif (!path) return;\n\t\t\t\n\t\t\tswitch(action) {\n\t\t\t\tcase 'download':\n\t\t\t\t\tdownloadFile(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'view':\n\t\t\t\t\tviewFile(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'properties':\n\t\t\t\t\tshowFileProperties(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'export-metadata':\n\t\t\t\t\texportMetadata(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'delete':\n\t\t\t\t\tconst fileName = path.split('/').pop();\n\t\t\t\t\tshowDeleteConfirm(fileName, function() {\n\t\t\t\t\t\tdeleteFile(path);\n\t\t\t\t\t}, `Are you sure you want to delete \"${fileName}\"? This action cannot be undone.`);\n\t\t\t\t\tbreak;\n\t\t\t}\n\t\t});\n\t\t\n\t\t// Initialize file manager event handlers from admin.js\n\t\tif (typeof setupFileManagerEventHandlers === 'function') {\n\t\t\tsetupFileManagerEventHandlers();\n\t\t}\n\t});\n\t\n\t// File browser specific functions\n\tfunction downloadFile(path) {\n\t\t// Open download URL in new tab\n\t\twindow.open((window.__BASE_PATH__ || '') + '/api/files/download?path=' + encodeURIComponent(path), '_blank');\n\t}\n\n\tfunction viewFile(path) {\n\t\t// Open file viewer in new tab\n\t\twindow.open((window.__BASE_PATH__ || '') + '/api/files/view?path=' + encodeURIComponent(path), '_blank');\n\t}\n\n\tfunction exportMetadata(path) {\n\t\t// Download metadata as an fs.meta.load-compatible .meta.gz\n\t\twindow.open((window.__BASE_PATH__ || '') + '/api/files/metadata?path=' + encodeURIComponent(path), '_blank');\n\t}\n\n\tfunction showFileProperties(path) {\n\t\t// Fetch file properties and show in modal\n\t\tfetch((window.__BASE_PATH__ || '') + '/api/files/properties?path=' + encodeURIComponent(path))\n\t\t\t.then(response => response.json())\n\t\t\t.then(data => {\n\t\t\t\tif (data.error) {\n\t\t\t\t\tshowAlert('Error loading file properties: ' + data.error, 'error');\n\t\t\t\t} else {\n\t\t\t\t\tdisplayFileProperties(data);\n\t\t\t\t}\n\t\t\t})\n\t\t\t.catch(error => {\n\t\t\t\tconsole.error('Error fetching file properties:', error);\n\t\t\t\tshowAlert('Error loading file properties: ' + error.message, 'error');\n\t\t\t});\n\t}\n\t\n\tfunction displayFileProperties(data) {\n\t\t// Create a comprehensive modal for file properties\n\t\tconst modalHtml = '<div class=\"modal fade\" id=\"filePropertiesModal\" tabindex=\"-1\">' +\n\t\t\t'<div class=\"modal-dialog modal-lg\">' +\n\t\t\t'<div class=\"modal-content\">' +\n\t\t\t'<div class=\"modal-header\">' +\n\t\t\t'<h5 class=\"modal-title\"><i class=\"bi bi-info-circle me-2\"></i>Properties: ' + (data.name || 'Unknown') + '</h5>' +\n\t\t\t'<button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\"></button>' +\n\t\t\t'</div>' +\n\t\t\t'<div class=\"modal-body\">' +\n\t\t\tcreateFilePropertiesContent(data) +\n\t\t\t'</div>' +\n\t\t\t'<div class=\"modal-footer\">' +\n\t\t\t'<button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Close</button>' Line truncated
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, "\"><div class=\"mb-3\"><label for=\"fileInput\" class=\"form-label\">Select Files</label> <input type=\"file\" class=\"form-control\" id=\"fileInput\" name=\"files\" multiple required><div class=\"form-text\">Choose one or more files to upload to the current directory. You can select multiple files by holding Ctrl (Cmd on Mac) while clicking. ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if data.UploadMaxSize > 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 95, "Total upload size limit: ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var41 string
|
||||
templ_7745c5c3_Var41, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(data.UploadMaxSize))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 320, Col: 67}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var41))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, ".")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 97, "</div></div><input type=\"hidden\" id=\"uploadPath\" name=\"path\" value=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var42 string
|
||||
templ_7745c5c3_Var42, templ_7745c5c3_Err = templruntime.NewAttributeExpression(data.CurrentPath).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 324, Col: 79}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var42)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 98, "\"><!-- File List Preview --><div id=\"fileListPreview\" class=\"mb-3\" style=\"display: none;\"><label class=\"form-label\">Selected Files:</label><div id=\"selectedFilesList\" class=\"border rounded p-2 bg-light\"><!-- Files will be listed here --></div></div><!-- Upload Progress --><div class=\"mb-3\" id=\"uploadProgress\" style=\"display: none;\"><label class=\"form-label\">Upload Progress:</label><div class=\"progress mb-2\"><div class=\"progress-bar progress-bar-striped progress-bar-animated\" role=\"progressbar\" style=\"width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\">0%</div></div><div id=\"uploadStatus\" class=\"small text-muted\">Preparing upload...</div></div></form></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-primary\" onclick=\"submitUploadFile()\"><i class=\"bi bi-upload me-1\"></i>Upload Files</button></div></div></div></div><!-- JavaScript for file browser functionality --><script>\n\tdocument.addEventListener('DOMContentLoaded', function() {\n\t\t// Format permissions in the main table\n\t\tdocument.querySelectorAll('.permissions-display').forEach(element => {\n\t\t\tconst mode = element.getAttribute('data-mode');\n\t\t\tconst isDirectory = element.getAttribute('data-is-directory') === 'true';\n\t\t\tif (mode) {\n\t\t\t\telement.textContent = formatPermissions(mode, isDirectory);\n\t\t\t}\n\t\t});\n\t\t\n\t\t// Handle file browser action buttons (download, view, properties, export-metadata, delete)\n\t\tdocument.addEventListener('click', function(e) {\n\t\t\tconst button = e.target.closest('[data-action]');\n\t\t\tif (!button) return;\n\t\t\t\n\t\t\tconst action = button.getAttribute('data-action');\n\t\t\tif (action === 'copy-url') {\n\t\t\t\tconst url = button.getAttribute('data-url');\n\t\t\t\tif (url) adminCopyToClipboard(url);\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tconst path = button.getAttribute('data-path');\n\n\t\t\tif (!path) return;\n\t\t\t\n\t\t\tswitch(action) {\n\t\t\t\tcase 'download':\n\t\t\t\t\tdownloadFile(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'view':\n\t\t\t\t\tviewFile(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'properties':\n\t\t\t\t\tshowFileProperties(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'export-metadata':\n\t\t\t\t\texportMetadata(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'delete':\n\t\t\t\t\tconst fileName = path.split('/').pop();\n\t\t\t\t\tshowDeleteConfirm(fileName, function() {\n\t\t\t\t\t\tdeleteFile(path);\n\t\t\t\t\t}, `Are you sure you want to delete \"${fileName}\"? This action cannot be undone.`);\n\t\t\t\t\tbreak;\n\t\t\t}\n\t\t});\n\t\t\n\t\t// Initialize file manager event handlers from admin.js\n\t\tif (typeof setupFileManagerEventHandlers === 'function') {\n\t\t\tsetupFileManagerEventHandlers();\n\t\t}\n\t});\n\t\n\t// File browser specific functions\n\tfunction downloadFile(path) {\n\t\t// Open download URL in new tab\n\t\twindow.open((window.__BASE_PATH__ || '') + '/api/files/download?path=' + encodeURIComponent(path), '_blank');\n\t}\n\n\tfunction viewFile(path) {\n\t\t// Open file viewer in new tab\n\t\twindow.open((window.__BASE_PATH__ || '') + '/api/files/view?path=' + encodeURIComponent(path), '_blank');\n\t}\n\n\tfunction exportMetadata(path) {\n\t\t// Download metadata as an fs.meta.load-compatible .meta.gz\n\t\twindow.open((window.__BASE_PATH__ || '') + '/api/files/metadata?path=' + encodeURIComponent(path), '_blank');\n\t}\n\n\tfunction showFileProperties(path) {\n\t\t// Fetch file properties and show in modal\n\t\tfetch((window.__BASE_PATH__ || '') + '/api/files/properties?path=' + encodeURIComponent(path))\n\t\t\t.then(response => response.json())\n\t\t\t.then(data => {\n\t\t\t\tif (data.error) {\n\t\t\t\t\tshowAlert('Error loading file properties: ' + data.error, 'error');\n\t\t\t\t} else {\n\t\t\t\t\tdisplayFileProperties(data);\n\t\t\t\t}\n\t\t\t})\n\t\t\t.catch(error => {\n\t\t\t\tconsole.error('Error fetching file properties:', error);\n\t\t\t\tshowAlert('Error loading file properties: ' + error.message, 'error');\n\t\t\t});\n\t}\n\t\n\tfunction displayFileProperties(data) {\n\t\t// Create a comprehensive modal for file properties\n\t\tconst modalHtml = '<div class=\"modal fade\" id=\"filePropertiesModal\" tabindex=\"-1\">' +\n\t\t\t'<div class=\"modal-dialog modal-lg\">' +\n\t\t\t'<div class=\"modal-content\">' +\n\t\t\t'<div class=\"modal-header\">' +\n\t\t\t'<h5 class=\"modal-title\"><i class=\"bi bi-info-circle me-2\"></i>Properties: ' + (data.name || 'Unknown') + '</h5>' +\n\t\t\t'<button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\"></button>' +\n\t\t\t'</div>' +\n\t\t\t'<div class=\"modal-body\">' +\n\t\t\tcreateFilePropertiesContent(data) +\n\t\t\t'</div>' +\n\t\t\t'<div class=\"modal-footer\">' +\n\t\t\t'<button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Close</button>' Line truncated
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
|
||||
)
|
||||
|
||||
templ S3TablesBuckets(data dash.S3TablesBucketsData) {
|
||||
@@ -160,6 +161,9 @@ templ S3TablesBuckets(data dash.S3TablesBucketsData) {
|
||||
<td>{ bucket.Name }</td>
|
||||
<td>
|
||||
<span class={ formatBadgeClass(bucket.Format) } title={ formatBadgeTitle(bucket.Format) }>{ formatLabel(bucket.Format) }</span>
|
||||
if s3vectors.IsVectorBucketFormat(bucket.Format) {
|
||||
<a class="ms-1 small" href={ dash.PUrl(ctx, "/object-store/vector-buckets/"+url.PathEscape(bucket.Name)) } title="This bucket's S3 Vectors indexes and policy">vector bucket</a>
|
||||
}
|
||||
</td>
|
||||
<td>{ bucket.OwnerAccountID }</td>
|
||||
<td class="text-muted small">{ bucket.ARN }</td>
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
|
||||
)
|
||||
|
||||
func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
|
||||
@@ -46,7 +47,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
|
||||
var templ_7745c5c3_Var2 string
|
||||
templ_7745c5c3_Var2, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.SafeURL(fmt.Sprintf("//localhost:%d/v1/config", data.IcebergPort))).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 24, Col: 119}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 25, Col: 119}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var2)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -65,7 +66,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
|
||||
var templ_7745c5c3_Var3 string
|
||||
templ_7745c5c3_Var3, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.SafeURL(fmt.Sprintf("//localhost:%d/v1/table", data.LancePort))).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 31, Col: 114}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 32, Col: 114}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var3)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -83,7 +84,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
|
||||
var templ_7745c5c3_Var4 string
|
||||
templ_7745c5c3_Var4, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.IcebergPort)).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 38, Col: 91}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 39, Col: 91}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var4)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -96,7 +97,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
|
||||
var templ_7745c5c3_Var5 string
|
||||
templ_7745c5c3_Var5, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.LancePort)).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 38, Col: 145}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 39, Col: 145}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var5)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -119,7 +120,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
|
||||
var templ_7745c5c3_Var6 string
|
||||
templ_7745c5c3_Var6, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.IcebergPort)).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 52, Col: 105}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 53, Col: 105}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var6)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -132,7 +133,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
|
||||
var templ_7745c5c3_Var7 string
|
||||
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("http://localhost:%d", data.IcebergPort))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 52, Col: 162}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 53, Col: 162}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -151,7 +152,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
|
||||
var templ_7745c5c3_Var8 string
|
||||
templ_7745c5c3_Var8, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.LancePort)).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 58, Col: 103}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 59, Col: 103}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var8)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -164,7 +165,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
|
||||
var templ_7745c5c3_Var9 string
|
||||
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("http://localhost:%d", data.LancePort))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 58, Col: 158}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 59, Col: 158}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -193,7 +194,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
|
||||
var templ_7745c5c3_Var10 string
|
||||
templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalBuckets))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 81, Col: 47}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 82, Col: 47}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -206,7 +207,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
|
||||
var templ_7745c5c3_Var11 string
|
||||
templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04"))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 100, Col: 54}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 101, Col: 54}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -220,7 +221,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
|
||||
var templ_7745c5c3_Var12 string
|
||||
templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 120, Col: 47}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 121, Col: 47}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -244,7 +245,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
|
||||
var templ_7745c5c3_Var13 string
|
||||
templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 160, Col: 28}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 161, Col: 28}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -283,7 +284,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
|
||||
var templ_7745c5c3_Var16 string
|
||||
templ_7745c5c3_Var16, templ_7745c5c3_Err = templruntime.NewAttributeExpression(formatBadgeTitle(bucket.Format)).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 162, Col: 99}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 163, Col: 99}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var16)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -310,220 +311,243 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
|
||||
var templ_7745c5c3_Var17 string
|
||||
templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(formatLabel(bucket.Format))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 162, Col: 130}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 163, Col: 130}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "</span></td><td>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "</span> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var18 string
|
||||
templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.OwnerAccountID)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 164, Col: 38}
|
||||
if s3vectors.IsVectorBucketFormat(bucket.Format) {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "<a class=\"ms-1 small\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var18 string
|
||||
templ_7745c5c3_Var18, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, "/object-store/vector-buckets/"+url.PathEscape(bucket.Name))).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 165, Col: 117}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var18)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "\" title=\"This bucket's S3 Vectors indexes and policy\">vector bucket</a>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "</td><td class=\"text-muted small\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "</td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var19 string
|
||||
templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
|
||||
templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.OwnerAccountID)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 165, Col: 52}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 168, Col: 38}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "</td><td><code class=\"small\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "</td><td class=\"text-muted small\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var20 string
|
||||
templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(bucketCatalogPath(bucket.Format, bucket.Name))
|
||||
templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 166, Col: 82}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 169, Col: 52}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "</code></td><td>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "</td><td><code class=\"small\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var21 string
|
||||
templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04"))
|
||||
templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(bucketCatalogPath(bucket.Format, bucket.Name))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 167, Col: 60}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 170, Col: 82}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "</td><td>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "</code></td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var22 string
|
||||
templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04"))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 171, Col: 60}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "</td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if bucket.PolicyStatementCount > 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "<button type=\"button\" class=\"badge bg-primary border-0 s3tables-bucket-policy-btn\" data-bucket-arn=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "<button type=\"button\" class=\"badge bg-primary border-0 s3tables-bucket-policy-btn\" data-bucket-arn=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var22 string
|
||||
templ_7745c5c3_Var22, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.ARN).Resolve(ctx)
|
||||
var templ_7745c5c3_Var23 string
|
||||
templ_7745c5c3_Var23, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.ARN).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 171, Col: 43}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 175, Col: 43}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var22)
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var23)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "\" title=\"View Bucket Policy\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "\" title=\"View Bucket Policy\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if bucket.PolicyStatementCount == 1 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "1 statement")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "1 statement")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
var templ_7745c5c3_Var23 string
|
||||
templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d statements", bucket.PolicyStatementCount))
|
||||
var templ_7745c5c3_Var24 string
|
||||
templ_7745c5c3_Var24, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d statements", bucket.PolicyStatementCount))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 175, Col: 74}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 179, Col: 74}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var24))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "</button>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "</button>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "<span class=\"text-muted\">Not configured</span>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "<span class=\"text-muted\">Not configured</span>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "</td><td><div class=\"btn-group btn-group-sm\" role=\"group\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "</td><td><div class=\"btn-group btn-group-sm\" role=\"group\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
bucketName, parseErr := s3tables.ParseBucketNameFromARN(bucket.ARN)
|
||||
if parseErr == nil {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "<a class=\"btn btn-outline-primary btn-sm\" href=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "<a class=\"btn btn-outline-primary btn-sm\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var24 string
|
||||
templ_7745c5c3_Var24, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName)))).ResolveURL(ctx)
|
||||
var templ_7745c5c3_Var25 string
|
||||
templ_7745c5c3_Var25, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName)))).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 186, Col: 166}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 190, Col: 166}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var24)
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var25)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "\"><i class=\"bi bi-folder2-open\"></i></a> ")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "\"><i class=\"bi bi-folder2-open\"></i></a> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "<button type=\"button\" class=\"btn btn-outline-primary btn-sm\" disabled title=\"Invalid bucket ARN\"><i class=\"bi bi-folder2-open\"></i></button> ")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "<button type=\"button\" class=\"btn btn-outline-primary btn-sm\" disabled title=\"Invalid bucket ARN\"><i class=\"bi bi-folder2-open\"></i></button> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "<button type=\"button\" class=\"btn btn-outline-success btn-sm s3tables-tags-btn\" data-resource-arn=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var25 string
|
||||
templ_7745c5c3_Var25, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.ARN).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 194, Col: 122}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var25)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "\" title=\"Tags\"><i class=\"bi bi-tags\"></i></button> <button type=\"button\" class=\"btn btn-outline-info btn-sm s3tables-bucket-policy-btn\" data-bucket-arn=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "<button type=\"button\" class=\"btn btn-outline-success btn-sm s3tables-tags-btn\" data-resource-arn=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var26 string
|
||||
templ_7745c5c3_Var26, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.ARN).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 197, Col: 126}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 198, Col: 122}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var26)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "\" title=\"Bucket Policy\"><i class=\"bi bi-shield\"></i></button> <button type=\"button\" class=\"btn btn-outline-danger btn-sm s3tables-delete-bucket-btn\" data-bucket-arn=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "\" title=\"Tags\"><i class=\"bi bi-tags\"></i></button> <button type=\"button\" class=\"btn btn-outline-info btn-sm s3tables-bucket-policy-btn\" data-bucket-arn=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var27 string
|
||||
templ_7745c5c3_Var27, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.ARN).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 200, Col: 128}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 201, Col: 126}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var27)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "\" data-bucket-name=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "\" title=\"Bucket Policy\"><i class=\"bi bi-shield\"></i></button> <button type=\"button\" class=\"btn btn-outline-danger btn-sm s3tables-delete-bucket-btn\" data-bucket-arn=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var28 string
|
||||
templ_7745c5c3_Var28, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.Name).Resolve(ctx)
|
||||
templ_7745c5c3_Var28, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.ARN).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 200, Col: 161}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 204, Col: 128}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var28)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "\" title=\"Delete\"><i class=\"bi bi-trash\"></i></button></div></td></tr>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "\" data-bucket-name=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var29 string
|
||||
templ_7745c5c3_Var29, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.Name).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 204, Col: 161}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var29)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "\" title=\"Delete\"><i class=\"bi bi-trash\"></i></button></div></td></tr>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
if len(data.Buckets) == 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "<tr><td colspan=\"8\" class=\"text-center text-muted py-4\"><i class=\"bi bi-table icon-3x mb-3 text-muted\"></i><div><h5>No table buckets found</h5><p>Create your first S3 Tables bucket to get started.</p><button type=\"button\" class=\"btn btn-primary\" data-bs-toggle=\"modal\" data-bs-target=\"#createS3TablesBucketModal\"><i class=\"bi bi-plus me-1\"></i>Create Bucket</button></div></td></tr>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "<tr><td colspan=\"8\" class=\"text-center text-muted py-4\"><i class=\"bi bi-table icon-3x mb-3 text-muted\"></i><div><h5>No table buckets found</h5><p>Create your first S3 Tables bucket to get started.</p><button type=\"button\" class=\"btn btn-primary\" data-bs-toggle=\"modal\" data-bs-target=\"#createS3TablesBucketModal\"><i class=\"bi bi-plus me-1\"></i>Create Bucket</button></div></td></tr>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "</tbody></table></div></div></div></div></div>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "</tbody></table></div></div></div></div></div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if data.IcebergPort > 0 || data.LancePort > 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "<div class=\"row\"><div class=\"col-12\"><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-code me-2\"></i>Client Examples</h6></div><div class=\"card-body\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "<div class=\"row\"><div class=\"col-12\"><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-code me-2\"></i>Client Examples</h6></div><div class=\"card-body\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if data.IcebergPort > 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "<h6><span class=\"badge bg-primary me-2\">ICEBERG</span>DuckDB</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-duckdb-example\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "<h6><span class=\"badge bg-primary me-2\">ICEBERG</span>DuckDB</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-duckdb-example\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var29 string
|
||||
templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(`INSTALL iceberg;
|
||||
var templ_7745c5c3_Var30 string
|
||||
templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(`INSTALL iceberg;
|
||||
LOAD iceberg;
|
||||
|
||||
CREATE SECRET (
|
||||
@@ -534,18 +558,18 @@ CREATE SECRET (
|
||||
|
||||
SELECT * FROM iceberg_scan('s3://my-table-bucket/my-namespace/my-table');`)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 251, Col: 74}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 255, Col: 74}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "</code></pre><h6 class=\"mt-4\"><span class=\"badge bg-primary me-2\">ICEBERG</span>Python (PyIceberg)</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-pyiceberg-example\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "</code></pre><h6 class=\"mt-4\"><span class=\"badge bg-primary me-2\">ICEBERG</span>Python (PyIceberg)</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-pyiceberg-example\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var30 string
|
||||
templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(`from pyiceberg.catalog import load_catalog
|
||||
var templ_7745c5c3_Var31 string
|
||||
templ_7745c5c3_Var31, templ_7745c5c3_Err = templ.JoinStringErrs(`from pyiceberg.catalog import load_catalog
|
||||
|
||||
catalog = load_catalog(
|
||||
name="seaweedfs",
|
||||
@@ -558,13 +582,13 @@ catalog = load_catalog(
|
||||
|
||||
namespaces = catalog.list_namespaces()`)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 268, Col: 39}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 272, Col: 39}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var31))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "</code></pre>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "</code></pre>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -574,25 +598,25 @@ namespaces = catalog.list_namespaces()`)
|
||||
templ_7745c5c3_ElementWriter := templ_7745c5c3_Buffer
|
||||
templ_7745c5c3_Buffer := templ.GetBuffer()
|
||||
var templ_7745c5c3_ElementScripts []templ.ComponentScript
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "<h6 class=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "<h6 class=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var31 = []any{templ.KV("mt-4", data.IcebergPort > 0)}
|
||||
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_ElementWriter, templ_7745c5c3_Var31...)
|
||||
var templ_7745c5c3_Var32 = []any{templ.KV("mt-4", data.IcebergPort > 0)}
|
||||
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_ElementWriter, templ_7745c5c3_Var32...)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var32 string
|
||||
templ_7745c5c3_Var32, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.CSSClasses(templ_7745c5c3_Var31).String()).Resolve(ctx)
|
||||
var templ_7745c5c3_Var33 string
|
||||
templ_7745c5c3_Var33, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.CSSClasses(templ_7745c5c3_Var32).String()).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 1, Col: 0}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var32)
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var33)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -606,30 +630,30 @@ namespaces = catalog.list_namespaces()`)
|
||||
}
|
||||
templ.ReleaseBuffer(templ_7745c5c3_Buffer)
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "><span class=\"badge bg-success me-2\">LANCE</span>Python (lance-namespace)</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-lance-namespace-example\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "><span class=\"badge bg-success me-2\">LANCE</span>Python (lance-namespace)</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-lance-namespace-example\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var33 string
|
||||
templ_7745c5c3_Var33, templ_7745c5c3_Err = templ.JoinStringErrs(`from lance_namespace import connect
|
||||
var templ_7745c5c3_Var34 string
|
||||
templ_7745c5c3_Var34, templ_7745c5c3_Err = templ.JoinStringErrs(`from lance_namespace import connect
|
||||
|
||||
ns = connect("rest", {"uri": "http://localhost:` + fmt.Sprintf("%d", data.LancePort) + `"})
|
||||
|
||||
ns.list_namespaces(id=["my-table-bucket"])
|
||||
ns.describe_table(id=["my-table-bucket", "my-namespace", "my-table"])`)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 281, Col: 70}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 285, Col: 70}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var33))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var34))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "</code></pre><h6 class=\"mt-4\"><span class=\"badge bg-success me-2\">LANCE</span>Python (pylance)</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-pylance-example\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "</code></pre><h6 class=\"mt-4\"><span class=\"badge bg-success me-2\">LANCE</span>Python (pylance)</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-pylance-example\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var34 string
|
||||
templ_7745c5c3_Var34, templ_7745c5c3_Err = templ.JoinStringErrs(`import lance
|
||||
var templ_7745c5c3_Var35 string
|
||||
templ_7745c5c3_Var35, templ_7745c5c3_Err = templ.JoinStringErrs(`import lance
|
||||
from lance_namespace import connect
|
||||
|
||||
ns = connect("rest", {"uri": "http://localhost:` + fmt.Sprintf("%d", data.LancePort) + `"})
|
||||
@@ -638,84 +662,84 @@ table = ns.describe_table(id=["my-table-bucket", "my-namespace", "my-table"])
|
||||
# The namespace vends both the location and the credentials to read it.
|
||||
dataset = lance.dataset(table.location, storage_options=table.storage_options)`)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 294, Col: 79}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 298, Col: 79}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var34))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var35))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "</code></pre>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "</code></pre>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "<script>\n\t\t\t\t\t\t\t\tconst s3tablesExamplesHost = window.location.hostname;\n\t\t\t\t\t\t\t\tconst s3tablesExamples = [\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-duckdb-example'),\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-pyiceberg-example'),\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-lance-namespace-example'),\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-pylance-example')\n\t\t\t\t\t\t\t\t];\n\t\t\t\t\t\t\t\ts3tablesExamples.forEach(example => {\n\t\t\t\t\t\t\t\t\tif (!example) {\n\t\t\t\t\t\t\t\t\t\treturn;\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\texample.textContent = example.textContent.replaceAll('localhost', s3tablesExamplesHost);\n\t\t\t\t\t\t\t\t});\n\t\t\t\t\t\t\t</script></div></div></div></div>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "<script>\n\t\t\t\t\t\t\t\tconst s3tablesExamplesHost = window.location.hostname;\n\t\t\t\t\t\t\t\tconst s3tablesExamples = [\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-duckdb-example'),\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-pyiceberg-example'),\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-lance-namespace-example'),\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-pylance-example')\n\t\t\t\t\t\t\t\t];\n\t\t\t\t\t\t\t\ts3tablesExamples.forEach(example => {\n\t\t\t\t\t\t\t\t\tif (!example) {\n\t\t\t\t\t\t\t\t\t\treturn;\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\texample.textContent = example.textContent.replaceAll('localhost', s3tablesExamplesHost);\n\t\t\t\t\t\t\t\t});\n\t\t\t\t\t\t\t</script></div></div></div></div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "</div><div class=\"modal fade\" id=\"createS3TablesBucketModal\" tabindex=\"-1\" aria-labelledby=\"createS3TablesBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"createS3TablesBucketModalLabel\"><i class=\"bi bi-plus me-2\"></i>Create Table Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"createS3TablesBucketForm\"><div class=\"modal-body\"><div class=\"mb-3\"><label for=\"s3tablesBucketName\" class=\"form-label\">Bucket Name</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesBucketName\" name=\"name\" placeholder=\"table-bucket-name\" required></div><div class=\"mb-3\"><label class=\"form-label\">Table Format</label><div class=\"row g-2\" id=\"s3tablesBucketFormatPicker\"><div class=\"col-6\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "</div><div class=\"modal fade\" id=\"createS3TablesBucketModal\" tabindex=\"-1\" aria-labelledby=\"createS3TablesBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"createS3TablesBucketModalLabel\"><i class=\"bi bi-plus me-2\"></i>Create Table Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"createS3TablesBucketForm\"><div class=\"modal-body\"><div class=\"mb-3\"><label for=\"s3tablesBucketName\" class=\"form-label\">Bucket Name</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesBucketName\" name=\"name\" placeholder=\"table-bucket-name\" required></div><div class=\"mb-3\"><label class=\"form-label\">Table Format</label><div class=\"row g-2\" id=\"s3tablesBucketFormatPicker\"><div class=\"col-6\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if data.IcebergPort > 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatIceberg\" value=\"ICEBERG\" checked> ")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatIceberg\" value=\"ICEBERG\" checked> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatIceberg\" value=\"ICEBERG\" disabled> ")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatIceberg\" value=\"ICEBERG\" disabled> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "<label class=\"btn btn-outline-primary w-100 text-start p-3\" for=\"s3tablesBucketFormatIceberg\"><span class=\"d-block fw-semibold\"><i class=\"bi bi-snow me-1\"></i>Iceberg</span> <span class=\"d-block small text-muted mt-1\">Served over the Iceberg REST catalog. Spark, Trino, PyIceberg, DuckDB, OLake Go.</span></label></div><div class=\"col-6\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, "<label class=\"btn btn-outline-primary w-100 text-start p-3\" for=\"s3tablesBucketFormatIceberg\"><span class=\"d-block fw-semibold\"><i class=\"bi bi-snow me-1\"></i>Iceberg</span> <span class=\"d-block small text-muted mt-1\">Served over the Iceberg REST catalog. Spark, Trino, PyIceberg, DuckDB, OLake Go.</span></label></div><div class=\"col-6\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if data.LancePort > 0 && data.IcebergPort <= 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, " <input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatLance\" value=\"LANCE\" checked> ")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, " <input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatLance\" value=\"LANCE\" checked> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else if data.LancePort > 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatLance\" value=\"LANCE\"> ")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 68, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatLance\" value=\"LANCE\"> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatLance\" value=\"LANCE\" disabled> ")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 69, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatLance\" value=\"LANCE\" disabled> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, "<label class=\"btn btn-outline-success w-100 text-start p-3\" for=\"s3tablesBucketFormatLance\"><span class=\"d-block fw-semibold\"><i class=\"bi bi-bounding-box me-1\"></i>Lance</span> <span class=\"d-block small text-muted mt-1\">Served over the Lance Namespace API. pylance, LanceDB, Ray.</span></label></div></div><div class=\"form-text\" id=\"s3tablesBucketFormatHint\" data-iceberg-port=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var35 string
|
||||
templ_7745c5c3_Var35, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.IcebergPort)).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 363, Col: 115}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var35)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 68, "\" data-lance-port=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 70, "<label class=\"btn btn-outline-success w-100 text-start p-3\" for=\"s3tablesBucketFormatLance\"><span class=\"d-block fw-semibold\"><i class=\"bi bi-bounding-box me-1\"></i>Lance</span> <span class=\"d-block small text-muted mt-1\">Served over the Lance Namespace API. pylance, LanceDB, Ray.</span></label></div></div><div class=\"form-text\" id=\"s3tablesBucketFormatHint\" data-iceberg-port=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var36 string
|
||||
templ_7745c5c3_Var36, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.LancePort)).Resolve(ctx)
|
||||
templ_7745c5c3_Var36, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.IcebergPort)).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 363, Col: 169}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 367, Col: 115}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var36)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 69, "\">A bucket holds one format. Tables of the other are refused.</div></div><div class=\"mb-3\"><label for=\"s3tablesBucketOwner\" class=\"form-label\">Owner (Optional)</label> <select class=\"form-select\" id=\"s3tablesBucketOwner\" name=\"owner\"><option value=\"\">No owner (admin-only access)</option></select><div class=\"form-text\">The S3 identity that owns this table bucket. Non-admin users can only access table buckets they own.</div></div><div class=\"mb-3\"><label for=\"s3tablesBucketTags\" class=\"form-label\">Tags</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesBucketTags\" name=\"tags\" placeholder=\"key1=value1,key2=value2\"><div class=\"form-text\">Optional tags in key=value format.</div></div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"submit\" class=\"btn btn-primary\"><i class=\"bi bi-plus me-1\"></i>Create</button></div></form></div></div></div><div class=\"modal fade\" id=\"deleteS3TablesBucketModal\" tabindex=\"-1\" aria-labelledby=\"deleteS3TablesBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"deleteS3TablesBucketModalLabel\"><i class=\"bi bi-exclamation-triangle me-2 text-warning\"></i>Delete Table Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p>Are you sure you want to delete the table bucket <strong id=\"deleteS3TablesBucketName\"></strong>?</p></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-danger\" onclick=\"deleteS3TablesBucket()\"><i class=\"bi bi-trash me-1\"></i>Delete</button></div></div></div></div>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 71, "\" data-lance-port=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var37 string
|
||||
templ_7745c5c3_Var37, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.LancePort)).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 367, Col: 169}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var37)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 72, "\">A bucket holds one format. Tables of the other are refused.</div></div><div class=\"mb-3\"><label for=\"s3tablesBucketOwner\" class=\"form-label\">Owner (Optional)</label> <select class=\"form-select\" id=\"s3tablesBucketOwner\" name=\"owner\"><option value=\"\">No owner (admin-only access)</option></select><div class=\"form-text\">The S3 identity that owns this table bucket. Non-admin users can only access table buckets they own.</div></div><div class=\"mb-3\"><label for=\"s3tablesBucketTags\" class=\"form-label\">Tags</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesBucketTags\" name=\"tags\" placeholder=\"key1=value1,key2=value2\"><div class=\"form-text\">Optional tags in key=value format.</div></div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"submit\" class=\"btn btn-primary\"><i class=\"bi bi-plus me-1\"></i>Create</button></div></form></div></div></div><div class=\"modal fade\" id=\"deleteS3TablesBucketModal\" tabindex=\"-1\" aria-labelledby=\"deleteS3TablesBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"deleteS3TablesBucketModalLabel\"><i class=\"bi bi-exclamation-triangle me-2 text-warning\"></i>Delete Table Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p>Are you sure you want to delete the table bucket <strong id=\"deleteS3TablesBucketName\"></strong>?</p></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-danger\" onclick=\"deleteS3TablesBucket()\"><i class=\"bi bi-trash me-1\"></i>Delete</button></div></div></div></div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -723,7 +747,7 @@ dataset = lance.dataset(table.location, storage_options=table.storage_options)`)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 70, "<div class=\"modal fade\" id=\"s3tablesBucketPolicyModal\" tabindex=\"-1\" aria-labelledby=\"s3tablesBucketPolicyModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog modal-xl\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"s3tablesBucketPolicyModalLabel\"><i class=\"bi bi-shield me-2\"></i>Table Bucket Policy</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"s3tablesBucketPolicyForm\"><div class=\"modal-body\"><input type=\"hidden\" id=\"s3tablesBucketPolicyArn\" name=\"bucket_arn\"><ul class=\"nav nav-tabs\" role=\"tablist\"><li class=\"nav-item\" role=\"presentation\"><button class=\"nav-link active\" id=\"s3tablesBucketPolicyEditorTabBtn\" type=\"button\" data-bs-toggle=\"tab\" data-bs-target=\"#s3tablesBucketPolicyEditorTab\" role=\"tab\" aria-controls=\"s3tablesBucketPolicyEditorTab\" aria-selected=\"true\">Editor</button></li><li class=\"nav-item\" role=\"presentation\"><button class=\"nav-link\" id=\"s3tablesBucketPolicyJsonTabBtn\" type=\"button\" data-bs-toggle=\"tab\" data-bs-target=\"#s3tablesBucketPolicyJsonTab\" role=\"tab\" aria-controls=\"s3tablesBucketPolicyJsonTab\" aria-selected=\"false\">JSON</button></li></ul><div class=\"tab-content border border-top-0 rounded-bottom p-3 mb-3\"><div class=\"tab-pane fade show active\" id=\"s3tablesBucketPolicyEditorTab\" role=\"tabpanel\" aria-labelledby=\"s3tablesBucketPolicyEditorTabBtn\"><div id=\"s3tablesBucketPolicyEditorBody\"></div><button type=\"button\" class=\"btn btn-sm btn-outline-secondary\" id=\"s3tablesBucketPolicyAddStatementBtn\"><i class=\"bi bi-plus me-1\"></i>Add statement</button></div><div class=\"tab-pane fade\" id=\"s3tablesBucketPolicyJsonTab\" role=\"tabpanel\" aria-labelledby=\"s3tablesBucketPolicyJsonTabBtn\"><textarea class=\"form-control\" id=\"s3tablesBucketPolicyText\" name=\"policy\" rows=\"12\" placeholder=\"{ }\"></textarea></div></div><div class=\"form-text\">Provide a policy JSON; use Delete Policy to remove the policy.</div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Close</button> <button type=\"button\" class=\"btn btn-outline-danger\" id=\"s3tablesBucketPolicyDeleteBtn\" onclick=\"deleteS3TablesBucketPolicy()\"><i class=\"bi bi-trash me-1\"></i>Delete Policy</button><!-- type=\"button\": a submit button would make Enter in any\n\t\t\t\t\t\t single-line editor input implicitly submit a half-built\n\t\t\t\t\t\t policy, which the backend stores verbatim. --><button type=\"button\" class=\"btn btn-primary\" id=\"s3tablesBucketPolicySaveBtn\"><i class=\"bi bi-floppy me-1\"></i>Save Policy</button></div></form></div></div></div><div class=\"modal fade\" id=\"s3tablesTagsModal\" tabindex=\"-1\" aria-labelledby=\"s3tablesTagsModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog modal-lg\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"s3tablesTagsModalLabel\"><i class=\"bi bi-tags me-2\"></i>Resource Tags</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"s3tablesTagsForm\"><div class=\"modal-body\"><input type=\"hidden\" id=\"s3tablesTagsResourceArn\" name=\"resource_arn\"><div class=\"mb-3\"><label class=\"form-label\">Existing Tags</label><pre class=\"bg-light p-3 border rounded\" id=\"s3tablesTagsList\">Loading...</pre></div><div class=\"mb-3\"><label for=\"s3tablesTagsInput\" class=\"form-label\">Add or Update Tags</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesTagsInput\" placeholder=\"key1=value1,key2=value2\"></div><div class=\"mb-3\"><label for=\"s3tablesTagsDeleteInput\" class=\"form-label\">Remove Tag Keys</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesTagsDeleteInput\" placeholder=\"key1,key2\"></div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Close</button> <button type=\"button\" class=\"btn btn-outline-danger\" onclick=\"deleteS3TablesTags()\"><i class=\"bi bi-trash me-1\"></i>Remove Tags</button> <button type=\"submit\" class=\"btn btn-primary\"><i class=\"bi bi-floppy me-1\"></i>Update Tags</button></div></form></div></div></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function() {\n\t\t\tinitS3TablesBuckets();\n\t\t});\n\t</script>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 73, "<div class=\"modal fade\" id=\"s3tablesBucketPolicyModal\" tabindex=\"-1\" aria-labelledby=\"s3tablesBucketPolicyModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog modal-xl\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"s3tablesBucketPolicyModalLabel\"><i class=\"bi bi-shield me-2\"></i>Table Bucket Policy</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"s3tablesBucketPolicyForm\"><div class=\"modal-body\"><input type=\"hidden\" id=\"s3tablesBucketPolicyArn\" name=\"bucket_arn\"><ul class=\"nav nav-tabs\" role=\"tablist\"><li class=\"nav-item\" role=\"presentation\"><button class=\"nav-link active\" id=\"s3tablesBucketPolicyEditorTabBtn\" type=\"button\" data-bs-toggle=\"tab\" data-bs-target=\"#s3tablesBucketPolicyEditorTab\" role=\"tab\" aria-controls=\"s3tablesBucketPolicyEditorTab\" aria-selected=\"true\">Editor</button></li><li class=\"nav-item\" role=\"presentation\"><button class=\"nav-link\" id=\"s3tablesBucketPolicyJsonTabBtn\" type=\"button\" data-bs-toggle=\"tab\" data-bs-target=\"#s3tablesBucketPolicyJsonTab\" role=\"tab\" aria-controls=\"s3tablesBucketPolicyJsonTab\" aria-selected=\"false\">JSON</button></li></ul><div class=\"tab-content border border-top-0 rounded-bottom p-3 mb-3\"><div class=\"tab-pane fade show active\" id=\"s3tablesBucketPolicyEditorTab\" role=\"tabpanel\" aria-labelledby=\"s3tablesBucketPolicyEditorTabBtn\"><div id=\"s3tablesBucketPolicyEditorBody\"></div><button type=\"button\" class=\"btn btn-sm btn-outline-secondary\" id=\"s3tablesBucketPolicyAddStatementBtn\"><i class=\"bi bi-plus me-1\"></i>Add statement</button></div><div class=\"tab-pane fade\" id=\"s3tablesBucketPolicyJsonTab\" role=\"tabpanel\" aria-labelledby=\"s3tablesBucketPolicyJsonTabBtn\"><textarea class=\"form-control\" id=\"s3tablesBucketPolicyText\" name=\"policy\" rows=\"12\" placeholder=\"{ }\"></textarea></div></div><div class=\"form-text\">Provide a policy JSON; use Delete Policy to remove the policy.</div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Close</button> <button type=\"button\" class=\"btn btn-outline-danger\" id=\"s3tablesBucketPolicyDeleteBtn\" onclick=\"deleteS3TablesBucketPolicy()\"><i class=\"bi bi-trash me-1\"></i>Delete Policy</button><!-- type=\"button\": a submit button would make Enter in any\n\t\t\t\t\t\t single-line editor input implicitly submit a half-built\n\t\t\t\t\t\t policy, which the backend stores verbatim. --><button type=\"button\" class=\"btn btn-primary\" id=\"s3tablesBucketPolicySaveBtn\"><i class=\"bi bi-floppy me-1\"></i>Save Policy</button></div></form></div></div></div><div class=\"modal fade\" id=\"s3tablesTagsModal\" tabindex=\"-1\" aria-labelledby=\"s3tablesTagsModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog modal-lg\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"s3tablesTagsModalLabel\"><i class=\"bi bi-tags me-2\"></i>Resource Tags</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"s3tablesTagsForm\"><div class=\"modal-body\"><input type=\"hidden\" id=\"s3tablesTagsResourceArn\" name=\"resource_arn\"><div class=\"mb-3\"><label class=\"form-label\">Existing Tags</label><pre class=\"bg-light p-3 border rounded\" id=\"s3tablesTagsList\">Loading...</pre></div><div class=\"mb-3\"><label for=\"s3tablesTagsInput\" class=\"form-label\">Add or Update Tags</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesTagsInput\" placeholder=\"key1=value1,key2=value2\"></div><div class=\"mb-3\"><label for=\"s3tablesTagsDeleteInput\" class=\"form-label\">Remove Tag Keys</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesTagsDeleteInput\" placeholder=\"key1,key2\"></div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Close</button> <button type=\"button\" class=\"btn btn-outline-danger\" onclick=\"deleteS3TablesTags()\"><i class=\"bi bi-trash me-1\"></i>Remove Tags</button> <button type=\"submit\" class=\"btn btn-primary\"><i class=\"bi bi-floppy me-1\"></i>Update Tags</button></div></form></div></div></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function() {\n\t\t\tinitS3TablesBuckets();\n\t\t});\n\t</script>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
)
|
||||
|
||||
templ S3VectorsBucketDetails(data dash.S3VectorsBucketData) {
|
||||
<div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
|
||||
<div>
|
||||
<h1 class="h2 mb-0">
|
||||
<i class="bi bi-bounding-box-circles me-2"></i>{ data.Bucket.Name }
|
||||
</h1>
|
||||
<nav aria-label="breadcrumb">
|
||||
<ol class="breadcrumb mb-0 mt-1">
|
||||
<li class="breadcrumb-item"><a href={ dash.PUrl(ctx, "/object-store/vector-buckets") }>Vector Buckets</a></li>
|
||||
<li class="breadcrumb-item active" aria-current="page">{ data.Bucket.Name }</li>
|
||||
</ol>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card shadow mb-4">
|
||||
<div class="card-body">
|
||||
<dl class="row mb-0">
|
||||
<dt class="col-sm-2">ARN</dt>
|
||||
<dd class="col-sm-10"><code>{ data.Bucket.ARN }</code></dd>
|
||||
<dt class="col-sm-2">Created</dt>
|
||||
<dd class="col-sm-10">{ data.Bucket.CreatedAt.Format("2006-01-02 15:04:05") }</dd>
|
||||
<dt class="col-sm-2">Encryption</dt>
|
||||
<dd class="col-sm-10">
|
||||
if data.Bucket.Encryption != "" {
|
||||
<span class="badge bg-success">{ data.Bucket.Encryption }</span>
|
||||
} else {
|
||||
<span class="text-muted">None</span>
|
||||
}
|
||||
</dd>
|
||||
</dl>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card shadow mb-4">
|
||||
<div class="card-header py-3">
|
||||
<h6 class="m-0 font-weight-bold text-primary">
|
||||
<i class="bi bi-list-ul me-2"></i>{ fmt.Sprintf("%d indexes", data.TotalIndexes) }
|
||||
if data.Truncated {
|
||||
<span class="text-muted small ms-2">{ fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalIndexes) }</span>
|
||||
}
|
||||
</h6>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="table-responsive">
|
||||
<table class="table table-hover" width="100%" cellspacing="0" id="s3vectorsIndexesTable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Data Type</th>
|
||||
<th>Dimension</th>
|
||||
<th>Distance Metric</th>
|
||||
<th>Non-filterable Metadata Keys</th>
|
||||
<th>Created</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
for _, index := range data.Indexes {
|
||||
<tr>
|
||||
<td>
|
||||
{ index.IndexName }
|
||||
<div class="text-muted small">{ index.IndexArn }</div>
|
||||
</td>
|
||||
<td>{ index.DataType }</td>
|
||||
<td>{ fmt.Sprintf("%d", index.Dimension) }</td>
|
||||
<td>{ index.DistanceMetric }</td>
|
||||
<td>
|
||||
if index.MetadataConfiguration != nil && len(index.MetadataConfiguration.NonFilterableMetadataKeys) > 0 {
|
||||
<code>{ strings.Join(index.MetadataConfiguration.NonFilterableMetadataKeys, ", ") }</code>
|
||||
} else {
|
||||
<span class="text-muted">None</span>
|
||||
}
|
||||
</td>
|
||||
<td>{ index.CreationTime.Format("2006-01-02 15:04") }</td>
|
||||
<td>
|
||||
<button type="button" class="btn btn-outline-danger btn-sm s3vectors-delete-index-btn" data-index-name={ index.IndexName } title="Delete">
|
||||
<i class="bi bi-trash"></i>
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
}
|
||||
if len(data.Indexes) == 0 {
|
||||
<tr>
|
||||
<td colspan="7" class="text-center text-muted py-4">
|
||||
No indexes. Create one with an S3 Vectors client, e.g. <code>aws s3vectors create-index</code>.
|
||||
</td>
|
||||
</tr>
|
||||
}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card shadow mb-4">
|
||||
<div class="card-header py-3">
|
||||
<h6 class="m-0 font-weight-bold text-primary">
|
||||
<i class="bi bi-shield me-2"></i>Bucket Policy
|
||||
</h6>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
if data.Policy != "" {
|
||||
<pre class="bg-light p-3 border rounded mb-0" id="s3vectorsBucketPolicy">{ indentJSON(data.Policy) }</pre>
|
||||
} else {
|
||||
<span class="text-muted">No policy. Set one with <code>PutVectorBucketPolicy</code>.</span>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal fade" id="deleteS3VectorsIndexModal" tabindex="-1" aria-labelledby="deleteS3VectorsIndexModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="deleteS3VectorsIndexModalLabel">
|
||||
<i class="bi bi-exclamation-triangle me-2 text-warning"></i>Delete Index
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<p>Delete the index <strong id="deleteS3VectorsIndexName"></strong> and all its vectors?</p>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="button" class="btn btn-danger" id="deleteS3VectorsIndexBtn">
|
||||
<i class="bi bi-trash me-1"></i>Delete
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="s3vectorsBucketDetails" data-bucket-name={ data.Bucket.Name }></div>
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', function () {
|
||||
const bucket = document.getElementById('s3vectorsBucketDetails').dataset.bucketName;
|
||||
const modalEl = document.getElementById('deleteS3VectorsIndexModal');
|
||||
const modal = new bootstrap.Modal(modalEl);
|
||||
document.querySelectorAll('.s3vectors-delete-index-btn').forEach(button => {
|
||||
button.addEventListener('click', function () {
|
||||
modalEl.dataset.indexName = this.dataset.indexName || '';
|
||||
document.getElementById('deleteS3VectorsIndexName').textContent = this.dataset.indexName || '';
|
||||
modal.show();
|
||||
});
|
||||
});
|
||||
document.getElementById('deleteS3VectorsIndexBtn').addEventListener('click', async function () {
|
||||
const index = modalEl.dataset.indexName;
|
||||
if (!bucket || !index) return;
|
||||
try {
|
||||
const response = await fetch(s3tBasePath('/api/s3vectors/indexes?bucket=' + encodeURIComponent(bucket) + '&index=' + encodeURIComponent(index)), { method: 'DELETE', headers: s3tWriteHeaders() });
|
||||
const data = await response.json();
|
||||
if (!response.ok) {
|
||||
alert(data.error || 'Failed to delete index');
|
||||
return;
|
||||
}
|
||||
location.reload();
|
||||
} catch (error) {
|
||||
alert('Failed to delete index: ' + error.message);
|
||||
}
|
||||
});
|
||||
});
|
||||
</script>
|
||||
}
|
||||
|
||||
const maxIndentedJSONDepth = 32
|
||||
|
||||
func indentJSON(s string) string {
|
||||
if jsonDepth(s) > maxIndentedJSONDepth {
|
||||
return s
|
||||
}
|
||||
var out bytes.Buffer
|
||||
if json.Indent(&out, []byte(s), "", " ") != nil {
|
||||
return s
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func jsonDepth(s string) int {
|
||||
depth, maxDepth := 0, 0
|
||||
inString, escaped := false, false
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
switch {
|
||||
case escaped:
|
||||
escaped = false
|
||||
case inString:
|
||||
if c == '\\' {
|
||||
escaped = true
|
||||
} else if c == '"' {
|
||||
inString = false
|
||||
}
|
||||
case c == '"':
|
||||
inString = true
|
||||
case c == '{' || c == '[':
|
||||
depth++
|
||||
if depth > maxDepth {
|
||||
maxDepth = depth
|
||||
}
|
||||
case c == '}' || c == ']':
|
||||
depth--
|
||||
}
|
||||
}
|
||||
return maxDepth
|
||||
}
|
||||
@@ -0,0 +1,392 @@
|
||||
// Code generated by templ - DO NOT EDIT.
|
||||
|
||||
// templ: version: v0.3.1100
|
||||
|
||||
package app
|
||||
|
||||
//lint:file-ignore SA4006 This context is only used if a nested component is present.
|
||||
|
||||
import "github.com/a-h/templ"
|
||||
import templruntime "github.com/a-h/templ/runtime"
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
)
|
||||
|
||||
func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
|
||||
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
|
||||
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
|
||||
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
|
||||
return templ_7745c5c3_CtxErr
|
||||
}
|
||||
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
|
||||
if !templ_7745c5c3_IsBuffer {
|
||||
defer func() {
|
||||
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err == nil {
|
||||
templ_7745c5c3_Err = templ_7745c5c3_BufErr
|
||||
}
|
||||
}()
|
||||
}
|
||||
ctx = templ.InitializeContext(ctx)
|
||||
ctx = templ.ClearChildren(ctx)
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom\"><div><h1 class=\"h2 mb-0\"><i class=\"bi bi-bounding-box-circles me-2\"></i>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var2 string
|
||||
templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.Name)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 16, Col: 69}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "</h1><nav aria-label=\"breadcrumb\"><ol class=\"breadcrumb mb-0 mt-1\"><li class=\"breadcrumb-item\"><a href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var3 string
|
||||
templ_7745c5c3_Var3, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, "/object-store/vector-buckets")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 20, Col: 89}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var3)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "\">Vector Buckets</a></li><li class=\"breadcrumb-item active\" aria-current=\"page\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var4 string
|
||||
templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.Name)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 21, Col: 78}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "</li></ol></nav></div></div><div class=\"card shadow mb-4\"><div class=\"card-body\"><dl class=\"row mb-0\"><dt class=\"col-sm-2\">ARN</dt><dd class=\"col-sm-10\"><code>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var5 string
|
||||
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.ARN)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 30, Col: 49}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "</code></dd><dt class=\"col-sm-2\">Created</dt><dd class=\"col-sm-10\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var6 string
|
||||
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.CreatedAt.Format("2006-01-02 15:04:05"))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 32, Col: 79}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "</dd><dt class=\"col-sm-2\">Encryption</dt><dd class=\"col-sm-10\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if data.Bucket.Encryption != "" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "<span class=\"badge bg-success\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var7 string
|
||||
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.Encryption)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 36, Col: 61}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "</span>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "<span class=\"text-muted\">None</span>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</dd></dl></div></div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-list-ul me-2\"></i>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var8 string
|
||||
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d indexes", data.TotalIndexes))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 47, Col: 84}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, " ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if data.Truncated {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "<span class=\"text-muted small ms-2\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var9 string
|
||||
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalIndexes))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 49, Col: 138}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "</span>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "</h6></div><div class=\"card-body\"><div class=\"table-responsive\"><table class=\"table table-hover\" width=\"100%\" cellspacing=\"0\" id=\"s3vectorsIndexesTable\"><thead><tr><th>Name</th><th>Data Type</th><th>Dimension</th><th>Distance Metric</th><th>Non-filterable Metadata Keys</th><th>Created</th><th>Actions</th></tr></thead> <tbody>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
for _, index := range data.Indexes {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "<tr><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var10 string
|
||||
templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(index.IndexName)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 71, Col: 26}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "<div class=\"text-muted small\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var11 string
|
||||
templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(index.IndexArn)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 72, Col: 55}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "</div></td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var12 string
|
||||
templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(index.DataType)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 74, Col: 28}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "</td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var13 string
|
||||
templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", index.Dimension))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 75, Col: 48}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "</td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var14 string
|
||||
templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(index.DistanceMetric)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 76, Col: 34}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "</td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if index.MetadataConfiguration != nil && len(index.MetadataConfiguration.NonFilterableMetadataKeys) > 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "<code>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var15 string
|
||||
templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(strings.Join(index.MetadataConfiguration.NonFilterableMetadataKeys, ", "))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 79, Col: 91}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "</code>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "<span class=\"text-muted\">None</span>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "</td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var16 string
|
||||
templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(index.CreationTime.Format("2006-01-02 15:04"))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 84, Col: 59}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "</td><td><button type=\"button\" class=\"btn btn-outline-danger btn-sm s3vectors-delete-index-btn\" data-index-name=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var17 string
|
||||
templ_7745c5c3_Var17, templ_7745c5c3_Err = templruntime.NewAttributeExpression(index.IndexName).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 86, Col: 129}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var17)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "\" title=\"Delete\"><i class=\"bi bi-trash\"></i></button></td></tr>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
if len(data.Indexes) == 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "<tr><td colspan=\"7\" class=\"text-center text-muted py-4\">No indexes. Create one with an S3 Vectors client, e.g. <code>aws s3vectors create-index</code>.</td></tr>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "</tbody></table></div></div></div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-shield me-2\"></i>Bucket Policy</h6></div><div class=\"card-body\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if data.Policy != "" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "<pre class=\"bg-light p-3 border rounded mb-0\" id=\"s3vectorsBucketPolicy\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var18 string
|
||||
templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(indentJSON(data.Policy))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 112, Col: 102}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "</pre>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "<span class=\"text-muted\">No policy. Set one with <code>PutVectorBucketPolicy</code>.</span>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "</div></div><div class=\"modal fade\" id=\"deleteS3VectorsIndexModal\" tabindex=\"-1\" aria-labelledby=\"deleteS3VectorsIndexModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"deleteS3VectorsIndexModalLabel\"><i class=\"bi bi-exclamation-triangle me-2 text-warning\"></i>Delete Index</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p>Delete the index <strong id=\"deleteS3VectorsIndexName\"></strong> and all its vectors?</p></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-danger\" id=\"deleteS3VectorsIndexBtn\"><i class=\"bi bi-trash me-1\"></i>Delete</button></div></div></div></div><div id=\"s3vectorsBucketDetails\" data-bucket-name=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var19 string
|
||||
templ_7745c5c3_Var19, templ_7745c5c3_Err = templruntime.NewAttributeExpression(data.Bucket.Name).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 139, Col: 69}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var19)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "\"></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function () {\n\t\t\tconst bucket = document.getElementById('s3vectorsBucketDetails').dataset.bucketName;\n\t\t\tconst modalEl = document.getElementById('deleteS3VectorsIndexModal');\n\t\t\tconst modal = new bootstrap.Modal(modalEl);\n\t\t\tdocument.querySelectorAll('.s3vectors-delete-index-btn').forEach(button => {\n\t\t\t\tbutton.addEventListener('click', function () {\n\t\t\t\t\tmodalEl.dataset.indexName = this.dataset.indexName || '';\n\t\t\t\t\tdocument.getElementById('deleteS3VectorsIndexName').textContent = this.dataset.indexName || '';\n\t\t\t\t\tmodal.show();\n\t\t\t\t});\n\t\t\t});\n\t\t\tdocument.getElementById('deleteS3VectorsIndexBtn').addEventListener('click', async function () {\n\t\t\t\tconst index = modalEl.dataset.indexName;\n\t\t\t\tif (!bucket || !index) return;\n\t\t\t\ttry {\n\t\t\t\t\tconst response = await fetch(s3tBasePath('/api/s3vectors/indexes?bucket=' + encodeURIComponent(bucket) + '&index=' + encodeURIComponent(index)), { method: 'DELETE', headers: s3tWriteHeaders() });\n\t\t\t\t\tconst data = await response.json();\n\t\t\t\t\tif (!response.ok) {\n\t\t\t\t\t\talert(data.error || 'Failed to delete index');\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\talert('Failed to delete index: ' + error.message);\n\t\t\t\t}\n\t\t\t});\n\t\t});\n\t</script>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
const maxIndentedJSONDepth = 32
|
||||
|
||||
func indentJSON(s string) string {
|
||||
if jsonDepth(s) > maxIndentedJSONDepth {
|
||||
return s
|
||||
}
|
||||
var out bytes.Buffer
|
||||
if json.Indent(&out, []byte(s), "", " ") != nil {
|
||||
return s
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func jsonDepth(s string) int {
|
||||
depth, maxDepth := 0, 0
|
||||
inString, escaped := false, false
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
switch {
|
||||
case escaped:
|
||||
escaped = false
|
||||
case inString:
|
||||
if c == '\\' {
|
||||
escaped = true
|
||||
} else if c == '"' {
|
||||
inString = false
|
||||
}
|
||||
case c == '"':
|
||||
inString = true
|
||||
case c == '{' || c == '[':
|
||||
depth++
|
||||
if depth > maxDepth {
|
||||
maxDepth = depth
|
||||
}
|
||||
case c == '}' || c == ']':
|
||||
depth--
|
||||
}
|
||||
}
|
||||
return maxDepth
|
||||
}
|
||||
|
||||
var _ = templruntime.GeneratedTemplate
|
||||
@@ -0,0 +1,189 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
)
|
||||
|
||||
templ S3VectorsBuckets(data dash.S3VectorsBucketsData) {
|
||||
<div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
|
||||
<h1 class="h2">
|
||||
<i class="bi bi-bounding-box-circles me-2"></i>Vector Buckets
|
||||
</h1>
|
||||
<div class="btn-toolbar mb-2 mb-md-0">
|
||||
<div class="btn-group me-2">
|
||||
<button type="button" class="btn btn-sm btn-primary" data-bs-toggle="modal" data-bs-target="#createS3VectorsBucketModal">
|
||||
<i class="bi bi-plus me-1"></i>Create Bucket
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="alert alert-info mb-4">
|
||||
<i class="bi bi-info-circle me-2"></i>
|
||||
Vector buckets serve the Amazon S3 Vectors API on the S3 port. Each one is a LANCE table bucket that keeps its indexes in its <code>vector</code> namespace.
|
||||
Indexes are created and filled with an S3 Vectors client, which needs a running <code>weed-vector</code> engine.
|
||||
</div>
|
||||
<div class="card shadow mb-4">
|
||||
<div class="card-header py-3">
|
||||
<h6 class="m-0 font-weight-bold text-primary">
|
||||
<i class="bi bi-bounding-box-circles me-2"></i>{ fmt.Sprintf("%d vector buckets", data.TotalBuckets) }
|
||||
if data.Truncated {
|
||||
<span class="text-muted small ms-2">{ fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalBuckets) }</span>
|
||||
}
|
||||
</h6>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="table-responsive">
|
||||
<table class="table table-hover" width="100%" cellspacing="0" id="s3vectorsBucketsTable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>ARN</th>
|
||||
<th>Encryption</th>
|
||||
<th>Created</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
for _, bucket := range data.Buckets {
|
||||
<tr>
|
||||
<td>
|
||||
<a href={ dash.PUrl(ctx, "/object-store/vector-buckets/"+url.PathEscape(bucket.Name)) }>{ bucket.Name }</a>
|
||||
</td>
|
||||
<td class="text-muted small">{ bucket.ARN }</td>
|
||||
<td>
|
||||
if bucket.Encryption != "" {
|
||||
<span class="badge bg-success">{ bucket.Encryption }</span>
|
||||
} else {
|
||||
<span class="text-muted">None</span>
|
||||
}
|
||||
</td>
|
||||
<td>{ bucket.CreatedAt.Format("2006-01-02 15:04") }</td>
|
||||
<td>
|
||||
<div class="btn-group btn-group-sm" role="group">
|
||||
<a class="btn btn-outline-primary btn-sm" href={ dash.PUrl(ctx, "/object-store/vector-buckets/"+url.PathEscape(bucket.Name)) } title="Indexes">
|
||||
<i class="bi bi-folder2-open"></i>
|
||||
</a>
|
||||
<button type="button" class="btn btn-outline-danger btn-sm s3vectors-delete-bucket-btn" data-bucket-name={ bucket.Name } title="Delete">
|
||||
<i class="bi bi-trash"></i>
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
}
|
||||
if len(data.Buckets) == 0 {
|
||||
<tr>
|
||||
<td colspan="5" class="text-center text-muted py-4">
|
||||
<i class="bi bi-bounding-box-circles icon-3x mb-3 text-muted"></i>
|
||||
<div>
|
||||
<h5>No vector buckets found</h5>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal fade" id="createS3VectorsBucketModal" tabindex="-1" aria-labelledby="createS3VectorsBucketModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="createS3VectorsBucketModalLabel">
|
||||
<i class="bi bi-plus me-2"></i>Create Vector Bucket
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<form id="createS3VectorsBucketForm">
|
||||
<div class="modal-body">
|
||||
<div class="mb-3">
|
||||
<label for="s3vectorsBucketName" class="form-label">Bucket Name</label>
|
||||
<input type="text" class="form-control" id="s3vectorsBucketName" name="name" placeholder="vector-bucket-name" required/>
|
||||
<div class="form-text">3 to 63 lowercase letters, digits and hyphens, starting and ending with a letter or digit.</div>
|
||||
</div>
|
||||
<div class="form-text">
|
||||
The bucket is created without encryption. To encrypt it, create it with an S3 Vectors client and <code>encryptionConfiguration</code>, which the S3 gateway checks against its SSE-S3 key.
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="submit" class="btn btn-primary">
|
||||
<i class="bi bi-plus me-1"></i>Create
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal fade" id="deleteS3VectorsBucketModal" tabindex="-1" aria-labelledby="deleteS3VectorsBucketModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="deleteS3VectorsBucketModalLabel">
|
||||
<i class="bi bi-exclamation-triangle me-2 text-warning"></i>Delete Vector Bucket
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<p>Delete the vector bucket <strong id="deleteS3VectorsBucketName"></strong>? A bucket that still has indexes is not deleted.</p>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="button" class="btn btn-danger" id="deleteS3VectorsBucketBtn">
|
||||
<i class="bi bi-trash me-1"></i>Delete
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', function () {
|
||||
const deleteModalEl = document.getElementById('deleteS3VectorsBucketModal');
|
||||
const deleteModal = new bootstrap.Modal(deleteModalEl);
|
||||
document.querySelectorAll('.s3vectors-delete-bucket-btn').forEach(button => {
|
||||
button.addEventListener('click', function () {
|
||||
deleteModalEl.dataset.bucketName = this.dataset.bucketName || '';
|
||||
document.getElementById('deleteS3VectorsBucketName').textContent = this.dataset.bucketName || '';
|
||||
deleteModal.show();
|
||||
});
|
||||
});
|
||||
document.getElementById('deleteS3VectorsBucketBtn').addEventListener('click', async function () {
|
||||
const bucket = deleteModalEl.dataset.bucketName;
|
||||
if (!bucket) return;
|
||||
try {
|
||||
const response = await fetch(s3tBasePath('/api/s3vectors/buckets?bucket=' + encodeURIComponent(bucket)), { method: 'DELETE', headers: s3tWriteHeaders() });
|
||||
const data = await response.json();
|
||||
if (!response.ok) {
|
||||
alert(data.error || 'Failed to delete vector bucket');
|
||||
return;
|
||||
}
|
||||
location.reload();
|
||||
} catch (error) {
|
||||
alert('Failed to delete vector bucket: ' + error.message);
|
||||
}
|
||||
});
|
||||
document.getElementById('createS3VectorsBucketForm').addEventListener('submit', async function (e) {
|
||||
e.preventDefault();
|
||||
const name = document.getElementById('s3vectorsBucketName').value.trim();
|
||||
try {
|
||||
const response = await fetch(s3tBasePath('/api/s3vectors/buckets'), {
|
||||
method: 'POST',
|
||||
headers: s3tWriteHeaders({ 'Content-Type': 'application/json' }),
|
||||
body: JSON.stringify({ name: name })
|
||||
});
|
||||
const data = await response.json();
|
||||
if (!response.ok) {
|
||||
alert(data.error || 'Failed to create vector bucket');
|
||||
return;
|
||||
}
|
||||
location.reload();
|
||||
} catch (error) {
|
||||
alert('Failed to create vector bucket: ' + error.message);
|
||||
}
|
||||
});
|
||||
});
|
||||
</script>
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
// Code generated by templ - DO NOT EDIT.
|
||||
|
||||
// templ: version: v0.3.1100
|
||||
|
||||
package app
|
||||
|
||||
//lint:file-ignore SA4006 This context is only used if a nested component is present.
|
||||
|
||||
import "github.com/a-h/templ"
|
||||
import templruntime "github.com/a-h/templ/runtime"
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
)
|
||||
|
||||
func S3VectorsBuckets(data dash.S3VectorsBucketsData) templ.Component {
|
||||
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
|
||||
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
|
||||
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
|
||||
return templ_7745c5c3_CtxErr
|
||||
}
|
||||
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
|
||||
if !templ_7745c5c3_IsBuffer {
|
||||
defer func() {
|
||||
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err == nil {
|
||||
templ_7745c5c3_Err = templ_7745c5c3_BufErr
|
||||
}
|
||||
}()
|
||||
}
|
||||
ctx = templ.InitializeContext(ctx)
|
||||
ctx = templ.ClearChildren(ctx)
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom\"><h1 class=\"h2\"><i class=\"bi bi-bounding-box-circles me-2\"></i>Vector Buckets</h1><div class=\"btn-toolbar mb-2 mb-md-0\"><div class=\"btn-group me-2\"><button type=\"button\" class=\"btn btn-sm btn-primary\" data-bs-toggle=\"modal\" data-bs-target=\"#createS3VectorsBucketModal\"><i class=\"bi bi-plus me-1\"></i>Create Bucket</button></div></div></div><div class=\"alert alert-info mb-4\"><i class=\"bi bi-info-circle me-2\"></i> Vector buckets serve the Amazon S3 Vectors API on the S3 port. Each one is a LANCE table bucket that keeps its indexes in its <code>vector</code> namespace. Indexes are created and filled with an S3 Vectors client, which needs a running <code>weed-vector</code> engine.</div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-bounding-box-circles me-2\"></i>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var2 string
|
||||
templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d vector buckets", data.TotalBuckets))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 31, Col: 104}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, " ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if data.Truncated {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "<span class=\"text-muted small ms-2\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var3 string
|
||||
templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalBuckets))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 33, Col: 138}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "</span>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "</h6></div><div class=\"card-body\"><div class=\"table-responsive\"><table class=\"table table-hover\" width=\"100%\" cellspacing=\"0\" id=\"s3vectorsBucketsTable\"><thead><tr><th>Name</th><th>ARN</th><th>Encryption</th><th>Created</th><th>Actions</th></tr></thead> <tbody>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
for _, bucket := range data.Buckets {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<tr><td><a href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var4 string
|
||||
templ_7745c5c3_Var4, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, "/object-store/vector-buckets/"+url.PathEscape(bucket.Name))).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 53, Col: 94}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var4)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var5 string
|
||||
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 53, Col: 110}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "</a></td><td class=\"text-muted small\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var6 string
|
||||
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 55, Col: 49}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "</td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if bucket.Encryption != "" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "<span class=\"badge bg-success\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var7 string
|
||||
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Encryption)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 58, Col: 60}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "</span>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "<span class=\"text-muted\">None</span>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "</td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var8 string
|
||||
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04"))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 63, Col: 57}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "</td><td><div class=\"btn-group btn-group-sm\" role=\"group\"><a class=\"btn btn-outline-primary btn-sm\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var9 string
|
||||
templ_7745c5c3_Var9, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, "/object-store/vector-buckets/"+url.PathEscape(bucket.Name))).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 66, Col: 134}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var9)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "\" title=\"Indexes\"><i class=\"bi bi-folder2-open\"></i></a> <button type=\"button\" class=\"btn btn-outline-danger btn-sm s3vectors-delete-bucket-btn\" data-bucket-name=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var10 string
|
||||
templ_7745c5c3_Var10, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.Name).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 69, Col: 128}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var10)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "\" title=\"Delete\"><i class=\"bi bi-trash\"></i></button></div></td></tr>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
if len(data.Buckets) == 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "<tr><td colspan=\"5\" class=\"text-center text-muted py-4\"><i class=\"bi bi-bounding-box-circles icon-3x mb-3 text-muted\"></i><div><h5>No vector buckets found</h5></div></td></tr>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "</tbody></table></div></div></div><div class=\"modal fade\" id=\"createS3VectorsBucketModal\" tabindex=\"-1\" aria-labelledby=\"createS3VectorsBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"createS3VectorsBucketModalLabel\"><i class=\"bi bi-plus me-2\"></i>Create Vector Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"createS3VectorsBucketForm\"><div class=\"modal-body\"><div class=\"mb-3\"><label for=\"s3vectorsBucketName\" class=\"form-label\">Bucket Name</label> <input type=\"text\" class=\"form-control\" id=\"s3vectorsBucketName\" name=\"name\" placeholder=\"vector-bucket-name\" required><div class=\"form-text\">3 to 63 lowercase letters, digits and hyphens, starting and ending with a letter or digit.</div></div><div class=\"form-text\">The bucket is created without encryption. To encrypt it, create it with an S3 Vectors client and <code>encryptionConfiguration</code>, which the S3 gateway checks against its SSE-S3 key.</div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"submit\" class=\"btn btn-primary\"><i class=\"bi bi-plus me-1\"></i>Create</button></div></form></div></div></div><div class=\"modal fade\" id=\"deleteS3VectorsBucketModal\" tabindex=\"-1\" aria-labelledby=\"deleteS3VectorsBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"deleteS3VectorsBucketModalLabel\"><i class=\"bi bi-exclamation-triangle me-2 text-warning\"></i>Delete Vector Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p>Delete the vector bucket <strong id=\"deleteS3VectorsBucketName\"></strong>? A bucket that still has indexes is not deleted.</p></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-danger\" id=\"deleteS3VectorsBucketBtn\"><i class=\"bi bi-trash me-1\"></i>Delete</button></div></div></div></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function () {\n\t\t\tconst deleteModalEl = document.getElementById('deleteS3VectorsBucketModal');\n\t\t\tconst deleteModal = new bootstrap.Modal(deleteModalEl);\n\t\t\tdocument.querySelectorAll('.s3vectors-delete-bucket-btn').forEach(button => {\n\t\t\t\tbutton.addEventListener('click', function () {\n\t\t\t\t\tdeleteModalEl.dataset.bucketName = this.dataset.bucketName || '';\n\t\t\t\t\tdocument.getElementById('deleteS3VectorsBucketName').textContent = this.dataset.bucketName || '';\n\t\t\t\t\tdeleteModal.show();\n\t\t\t\t});\n\t\t\t});\n\t\t\tdocument.getElementById('deleteS3VectorsBucketBtn').addEventListener('click', async function () {\n\t\t\t\tconst bucket = deleteModalEl.dataset.bucketName;\n\t\t\t\tif (!bucket) return;\n\t\t\t\ttry {\n\t\t\t\t\tconst response = await fetch(s3tBasePath('/api/s3vectors/buckets?bucket=' + encodeURIComponent(bucket)), { method: 'DELETE', headers: s3tWriteHeaders() });\n\t\t\t\t\tconst data = await response.json();\n\t\t\t\t\tif (!response.ok) {\n\t\t\t\t\t\talert(data.error || 'Failed to delete vector bucket');\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\talert('Failed to delete vector bucket: ' + error.message);\n\t\t\t\t}\n\t\t\t});\n\t\t\tdocument.getElementById('createS3VectorsBucketForm').addEventListener('submit', async function (e) {\n\t\t\t\te.preventDefault();\n\t\t\t\tconst name = document.getElementById('s3vectorsBucketName').value.trim();\n\t\t\t\ttry {\n\t\t\t\t\tconst response = await fetch(s3tBasePath('/api/s3vectors/buckets'), {\n\t\t\t\t\t\tmethod: 'POST',\n\t\t\t\t\t\theaders: s3tWriteHeaders({ 'Content-Type': 'application/json' }),\n\t\t\t\t\t\tbody: JSON.stringify({ name: name })\n\t\t\t\t\t});\n\t\t\t\t\tconst data = await response.json();\n\t\t\t\t\tif (!response.ok) {\n\t\t\t\t\t\talert(data.error || 'Failed to create vector bucket');\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\talert('Failed to create vector bucket: ' + error.message);\n\t\t\t\t}\n\t\t\t});\n\t\t});\n\t</script>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
var _ = templruntime.GeneratedTemplate
|
||||
@@ -0,0 +1,80 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestS3VectorsBucketsRendersBuckets(t *testing.T) {
|
||||
data := dash.S3VectorsBucketsData{TotalBuckets: 2, Buckets: []dash.S3VectorsBucketSummary{
|
||||
{Name: "media", ARN: "arn:aws:s3vectors:us-east-1:000000000000:bucket/media", CreatedAt: time.Unix(0, 0)},
|
||||
{Name: "sealed", ARN: "arn:aws:s3vectors:us-east-1:000000000000:bucket/sealed", Encryption: "AES256", CreatedAt: time.Unix(0, 0)},
|
||||
}}
|
||||
var out bytes.Buffer
|
||||
require.NoError(t, S3VectorsBuckets(data).Render(dash.WithURLPrefix(t.Context(), "/admin"), &out))
|
||||
page := out.String()
|
||||
require.Contains(t, page, `href="/admin/object-store/vector-buckets/media"`)
|
||||
require.Contains(t, page, `data-bucket-name="sealed"`)
|
||||
require.Contains(t, page, `<span class="badge bg-success">AES256</span>`)
|
||||
require.Contains(t, page, "2 vector buckets")
|
||||
require.NotContains(t, page, "No vector buckets found")
|
||||
require.NotContains(t, page, "showing the first")
|
||||
|
||||
data.Truncated = true
|
||||
out.Reset()
|
||||
require.NoError(t, S3VectorsBuckets(data).Render(t.Context(), &out))
|
||||
require.Contains(t, out.String(), "showing the first 2; list the rest with an S3 Vectors client")
|
||||
}
|
||||
|
||||
func TestS3VectorsBucketDetailsRendersIndexesAndPolicy(t *testing.T) {
|
||||
data := dash.S3VectorsBucketData{
|
||||
Bucket: dash.S3VectorsBucketSummary{Name: "media", ARN: "arn:aws:s3vectors:us-east-1:000000000000:bucket/media"},
|
||||
Policy: `{"Version":"2012-10-17","Statement":[]}`,
|
||||
TotalIndexes: 1,
|
||||
Indexes: []*s3vectors.Index{{
|
||||
IndexName: "movies", IndexArn: "arn:aws:s3vectors:us-east-1:000000000000:bucket/media/index/movies",
|
||||
DataType: "float32", Dimension: 384, DistanceMetric: "cosine",
|
||||
MetadataConfiguration: &s3vectors.MetadataConfiguration{NonFilterableMetadataKeys: []string{"summary", "body"}},
|
||||
}},
|
||||
}
|
||||
var out bytes.Buffer
|
||||
require.NoError(t, S3VectorsBucketDetails(data).Render(t.Context(), &out))
|
||||
page := out.String()
|
||||
for _, want := range []string{"movies", ">384<", ">cosine<", ">float32<", "summary, body", `data-index-name="movies"`, "{\n "Version": "2012-10-17",\n "Statement": []\n}"} {
|
||||
require.Contains(t, page, want)
|
||||
}
|
||||
require.False(t, strings.Contains(page, "No indexes."), "an index is listed")
|
||||
|
||||
data.Policy = "not json"
|
||||
out.Reset()
|
||||
require.NoError(t, S3VectorsBucketDetails(data).Render(t.Context(), &out))
|
||||
require.Contains(t, out.String(), ">not json</pre>")
|
||||
|
||||
data.Policy = `{"Version":"2012-10-17","Extra":` + strings.Repeat("[", 5000) + strings.Repeat("]", 5000) + "}"
|
||||
out.Reset()
|
||||
require.NoError(t, S3VectorsBucketDetails(data).Render(t.Context(), &out))
|
||||
require.Contains(t, out.String(), `"Extra":[[[[`, "a deeply nested policy is shown as stored")
|
||||
}
|
||||
|
||||
func TestJSONDepthSkipsStrings(t *testing.T) {
|
||||
require.Equal(t, 3, jsonDepth(`{"a":["[{\"[",{}]}`))
|
||||
require.Equal(t, 0, jsonDepth(`"[[["`))
|
||||
}
|
||||
|
||||
func TestS3TablesBucketsLinksVectorBuckets(t *testing.T) {
|
||||
data := dash.S3TablesBucketsData{Buckets: []dash.S3TablesBucketSummary{
|
||||
{Name: "vectors", ARN: "arn:aws:s3tables:us-east-1:000000000000:bucket/vectors", Format: "LANCE"},
|
||||
{Name: "warehouse", ARN: "arn:aws:s3tables:us-east-1:000000000000:bucket/warehouse", Format: "ICEBERG"},
|
||||
}}
|
||||
var out bytes.Buffer
|
||||
require.NoError(t, S3TablesBuckets(data).Render(t.Context(), &out))
|
||||
page := out.String()
|
||||
require.Contains(t, page, `href="/object-store/vector-buckets/vectors"`)
|
||||
require.NotContains(t, page, `href="/object-store/vector-buckets/warehouse"`)
|
||||
}
|
||||
@@ -189,6 +189,11 @@ templ Layout(view ViewContext, content templ.Component) {
|
||||
<i class="bi bi-table me-2"></i>Table Buckets
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href={ view.P("/object-store/vector-buckets") }>
|
||||
<i class="bi bi-bounding-box-circles me-2"></i>Vector Buckets
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href={ view.P("/object-store/users") }>
|
||||
<i class="bi bi-people me-2"></i>Users
|
||||
|
||||
@@ -534,365 +534,378 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var32 string
|
||||
templ_7745c5c3_Var32, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/users")).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var32, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/vector-buckets")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 193, Col: 84}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 193, Col: 93}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var32)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "\"><i class=\"bi bi-people me-2\"></i>Users</a></li><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "\"><i class=\"bi bi-bounding-box-circles me-2\"></i>Vector Buckets</a></li><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var33 string
|
||||
templ_7745c5c3_Var33, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/groups")).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var33, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/users")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 198, Col: 85}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 198, Col: 84}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var33)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "\"><i class=\"bi bi-people-fill me-2\"></i>Groups</a></li><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "\"><i class=\"bi bi-people me-2\"></i>Users</a></li><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var34 string
|
||||
templ_7745c5c3_Var34, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/service-accounts")).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var34, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/groups")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 203, Col: 95}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 203, Col: 85}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var34)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "\"><i class=\"bi bi-robot me-2\"></i>Service Accounts</a></li><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "\"><i class=\"bi bi-people-fill me-2\"></i>Groups</a></li><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var35 string
|
||||
templ_7745c5c3_Var35, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/policies")).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var35, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/service-accounts")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 208, Col: 87}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 208, Col: 95}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var35)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "\"><i class=\"bi bi-shield me-2\"></i>Policies</a></li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>MANAGEMENT</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "\"><i class=\"bi bi-robot me-2\"></i>Service Accounts</a></li><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var36 string
|
||||
templ_7745c5c3_Var36, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/files")).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var36, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/policies")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 219, Col: 71}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 213, Col: 87}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var36)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "\"><i class=\"bi bi-folder me-2\"></i>File Browser</a></li><li class=\"nav-item\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "\"><i class=\"bi bi-shield me-2\"></i>Policies</a></li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>MANAGEMENT</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var37 string
|
||||
templ_7745c5c3_Var37, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/files")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 224, Col: 71}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var37)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "\"><i class=\"bi bi-folder me-2\"></i>File Browser</a></li><li class=\"nav-item\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if isMQPage {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "<a class=\"nav-link\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#messageQueueSubmenu\" aria-expanded=\"true\" aria-controls=\"messageQueueSubmenu\"><i class=\"bi bi-chat-dots me-2\"></i>Message Queue <i class=\"bi bi-chevron-down ms-auto\"></i></a> ")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "<a class=\"nav-link\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#messageQueueSubmenu\" aria-expanded=\"true\" aria-controls=\"messageQueueSubmenu\"><i class=\"bi bi-chat-dots me-2\"></i>Message Queue <i class=\"bi bi-chevron-down ms-auto\"></i></a> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "<a class=\"nav-link collapsed\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#messageQueueSubmenu\" aria-expanded=\"false\" aria-controls=\"messageQueueSubmenu\"><i class=\"bi bi-chat-dots me-2\"></i>Message Queue <i class=\"bi bi-chevron-down ms-auto\"></i></a> ")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "<a class=\"nav-link collapsed\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#messageQueueSubmenu\" aria-expanded=\"false\" aria-controls=\"messageQueueSubmenu\"><i class=\"bi bi-chat-dots me-2\"></i>Message Queue <i class=\"bi bi-chevron-down ms-auto\"></i></a> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
if isMQPage {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "<div class=\"collapse show\" id=\"messageQueueSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "<div class=\"collapse show\" id=\"messageQueueSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if currentPath == "/mq/brokers" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "<a class=\"nav-link py-2 active\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var37 string
|
||||
templ_7745c5c3_Var37, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/brokers")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 240, Col: 108}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var37)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "\"><i class=\"bi bi-server me-2\"></i>Brokers</a>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "<a class=\"nav-link py-2\" href=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "<a class=\"nav-link py-2 active\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var38 string
|
||||
templ_7745c5c3_Var38, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/brokers")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 244, Col: 101}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 245, Col: 108}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var38)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "\"><i class=\"bi bi-server me-2\"></i>Brokers</a>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "\"><i class=\"bi bi-server me-2\"></i>Brokers</a>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "</li><li class=\"nav-item\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if currentPath == "/mq/topics" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "<a class=\"nav-link py-2 active\" href=\"")
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "<a class=\"nav-link py-2\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var39 string
|
||||
templ_7745c5c3_Var39, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/topics")).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var39, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/brokers")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 251, Col: 107}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 249, Col: 101}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var39)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "\"><i class=\"bi bi-card-list me-2\"></i>Topics</a>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "\"><i class=\"bi bi-server me-2\"></i>Brokers</a>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "<a class=\"nav-link py-2\" href=\"")
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "</li><li class=\"nav-item\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if currentPath == "/mq/topics" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "<a class=\"nav-link py-2 active\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var40 string
|
||||
templ_7745c5c3_Var40, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/topics")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 255, Col: 100}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 256, Col: 107}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var40)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "\"><i class=\"bi bi-card-list me-2\"></i>Topics</a>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "\"><i class=\"bi bi-card-list me-2\"></i>Topics</a>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "<a class=\"nav-link py-2\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var41 string
|
||||
templ_7745c5c3_Var41, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/topics")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 260, Col: 100}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var41)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "\"><i class=\"bi bi-card-list me-2\"></i>Topics</a>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "</li></ul></div>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "</li></ul></div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "<div class=\"collapse\" id=\"messageQueueSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var41 string
|
||||
templ_7745c5c3_Var41, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/brokers")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 267, Col: 97}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var41)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "\"><i class=\"bi bi-server me-2\"></i>Brokers</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "<div class=\"collapse\" id=\"messageQueueSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var42 string
|
||||
templ_7745c5c3_Var42, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/topics")).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var42, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/brokers")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 272, Col: 96}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 272, Col: 97}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var42)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "\"><i class=\"bi bi-card-list me-2\"></i>Topics</a></li></ul></div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "</li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>WORKERS</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if isDefaultWorkerPage {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "<a class=\"nav-link active\" href=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "\"><i class=\"bi bi-server me-2\"></i>Brokers</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var43 string
|
||||
templ_7745c5c3_Var43, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/default")).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var43, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/topics")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 289, Col: 97}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 277, Col: 96}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var43)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "\"><i class=\"bi bi-server me-2\"></i>Default</a>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "\"><i class=\"bi bi-card-list me-2\"></i>Topics</a></li></ul></div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "<a class=\"nav-link\" href=\"")
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "</li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>WORKERS</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if isDefaultWorkerPage {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "<a class=\"nav-link active\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var44 string
|
||||
templ_7745c5c3_Var44, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/default")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 293, Col: 90}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 294, Col: 97}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var44)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "\"><i class=\"bi bi-server me-2\"></i>Default</a>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "\"><i class=\"bi bi-server me-2\"></i>Default</a>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "</li><li class=\"nav-item\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if isLifecycleWorkerPage {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "<a class=\"nav-link active\" href=\"")
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "<a class=\"nav-link\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var45 string
|
||||
templ_7745c5c3_Var45, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/lifecycle")).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var45, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/default")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 300, Col: 99}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 298, Col: 90}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var45)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, "\"><i class=\"bi bi-hourglass-split me-2\"></i>Lifecycle</a>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "\"><i class=\"bi bi-server me-2\"></i>Default</a>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, "<a class=\"nav-link\" href=\"")
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "</li><li class=\"nav-item\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if isLifecycleWorkerPage {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, "<a class=\"nav-link active\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var46 string
|
||||
templ_7745c5c3_Var46, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/lifecycle")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 304, Col: 92}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 305, Col: 99}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var46)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, "\"><i class=\"bi bi-hourglass-split me-2\"></i>Lifecycle</a>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, "\"><i class=\"bi bi-hourglass-split me-2\"></i>Lifecycle</a>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 68, "</li><li class=\"nav-item\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if isIcebergWorkerPage {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 69, "<a class=\"nav-link active\" href=\"")
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, "<a class=\"nav-link\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var47 string
|
||||
templ_7745c5c3_Var47, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/iceberg")).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var47, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/lifecycle")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 311, Col: 97}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 309, Col: 92}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var47)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 70, "\"><i class=\"bi bi-snow me-2\"></i>Iceberg</a>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 68, "\"><i class=\"bi bi-hourglass-split me-2\"></i>Lifecycle</a>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 71, "<a class=\"nav-link\" href=\"")
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 69, "</li><li class=\"nav-item\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if isIcebergWorkerPage {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 70, "<a class=\"nav-link active\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var48 string
|
||||
templ_7745c5c3_Var48, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/iceberg")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 315, Col: 90}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 316, Col: 97}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var48)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 72, "\"><i class=\"bi bi-snow me-2\"></i>Iceberg</a>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 71, "\"><i class=\"bi bi-snow me-2\"></i>Iceberg</a>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 73, "</li><li class=\"nav-item\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if isLanceWorkerPage {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 74, "<a class=\"nav-link active\" href=\"")
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 72, "<a class=\"nav-link\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var49 string
|
||||
templ_7745c5c3_Var49, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/lance")).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var49, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/iceberg")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 322, Col: 95}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 320, Col: 90}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var49)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 75, "\"><i class=\"bi bi-bounding-box me-2\"></i>Lance</a>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 73, "\"><i class=\"bi bi-snow me-2\"></i>Iceberg</a>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 76, "<a class=\"nav-link\" href=\"")
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 74, "</li><li class=\"nav-item\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if isLanceWorkerPage {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 75, "<a class=\"nav-link active\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var50 string
|
||||
templ_7745c5c3_Var50, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/lance")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 326, Col: 88}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 327, Col: 95}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var50)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 77, "\"><i class=\"bi bi-bounding-box me-2\"></i>Lance</a>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 76, "\"><i class=\"bi bi-bounding-box me-2\"></i>Lance</a>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 77, "<a class=\"nav-link\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var51 string
|
||||
templ_7745c5c3_Var51, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/lance")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 331, Col: 88}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var51)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 78, "\"><i class=\"bi bi-bounding-box me-2\"></i>Lance</a>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 78, "</li></ul></div></div><!-- Sidebar backdrop for mobile --><div class=\"sidebar-backdrop\" id=\"sidebarBackdrop\"></div><!-- Main content --><main class=\"col-md-9 ms-sm-auto col-lg-10 px-3 px-md-4\"><div class=\"pt-3\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 79, "</li></ul></div></div><!-- Sidebar backdrop for mobile --><div class=\"sidebar-backdrop\" id=\"sidebarBackdrop\"></div><!-- Main content --><main class=\"col-md-9 ms-sm-auto col-lg-10 px-3 px-md-4\"><div class=\"pt-3\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -900,108 +913,95 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 79, "</div></main></div></div><!-- Footer --><footer class=\"footer mt-auto py-3 bg-light\"><div class=\"container-fluid text-center\"><small class=\"text-muted\">© ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var51 string
|
||||
templ_7745c5c3_Var51, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year()))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 351, Col: 60}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var51))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 80, " SeaweedFS Admin v")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 80, "</div></main></div></div><!-- Footer --><footer class=\"footer mt-auto py-3 bg-light\"><div class=\"container-fluid text-center\"><small class=\"text-muted\">© ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var52 string
|
||||
templ_7745c5c3_Var52, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER)
|
||||
templ_7745c5c3_Var52, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year()))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 351, Col: 102}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 356, Col: 60}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var52))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 81, " ")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 81, " SeaweedFS Admin v")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var53 string
|
||||
templ_7745c5c3_Var53, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 356, Col: 102}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var53))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 82, " ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if version.COMMIT != "" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 82, "<span class=\"mx-1\">(")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 83, "<span class=\"mx-1\">(")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var53 string
|
||||
templ_7745c5c3_Var53, templ_7745c5c3_Err = templ.JoinStringErrs(version.COMMIT)
|
||||
var templ_7745c5c3_Var54 string
|
||||
templ_7745c5c3_Var54, templ_7745c5c3_Err = templ.JoinStringErrs(version.COMMIT)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 353, Col: 55}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 358, Col: 55}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var53))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var54))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 83, ")</span> ")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 84, ")</span> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
if !strings.Contains(version.VERSION, "enterprise") {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 84, "<span class=\"mx-2\">•</span> <a href=\"https://seaweedfs.com\" target=\"_blank\" class=\"text-decoration-none\"><i class=\"bi bi-star me-1\"></i>Enterprise Version Available</a>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 85, "<span class=\"mx-2\">•</span> <a href=\"https://seaweedfs.com\" target=\"_blank\" class=\"text-decoration-none\"><i class=\"bi bi-star me-1\"></i>Enterprise Version Available</a>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 85, "</small></div></footer><!-- Bootstrap JS --><script src=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var54 string
|
||||
templ_7745c5c3_Var54, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/bootstrap.bundle.min.js"))).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 366, Col: 70}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var54)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 86, "\"></script><!-- Modal Alerts JS (replaces native alert/confirm) --><script src=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 86, "</small></div></footer><!-- Bootstrap JS --><script src=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var55 string
|
||||
templ_7745c5c3_Var55, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/modal-alerts.js"))).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var55, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/bootstrap.bundle.min.js"))).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 368, Col: 62}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 371, Col: 70}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var55)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 87, "\"></script><!-- Custom JS --><script src=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 87, "\"></script><!-- Modal Alerts JS (replaces native alert/confirm) --><script src=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var56 string
|
||||
templ_7745c5c3_Var56, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/admin.js"))).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var56, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/modal-alerts.js"))).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 370, Col: 55}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 373, Col: 62}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var56)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 88, "\"></script><script src=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 88, "\"></script><!-- Custom JS --><script src=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var57 string
|
||||
templ_7745c5c3_Var57, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/iam-utils.js"))).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var57, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/admin.js"))).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 371, Col: 59}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 375, Col: 55}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var57)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -1012,9 +1012,9 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var58 string
|
||||
templ_7745c5c3_Var58, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/s3tables.js"))).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var58, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/iam-utils.js"))).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 372, Col: 58}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 376, Col: 59}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var58)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -1025,15 +1025,28 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var59 string
|
||||
templ_7745c5c3_Var59, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/policy_editor.js"))).ResolveURL(ctx)
|
||||
templ_7745c5c3_Var59, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/s3tables.js"))).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 373, Col: 63}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 377, Col: 58}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var59)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 91, "\"></script></body></html>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 91, "\"></script><script src=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var60 string
|
||||
templ_7745c5c3_Var60, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/policy_editor.js"))).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 378, Col: 63}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var60)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 92, "\"></script></body></html>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -1059,134 +1072,134 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon
|
||||
ctx = templ.InitializeContext(ctx)
|
||||
ctx = templ.ClearChildren(ctx)
|
||||
prefix := dash.URLPrefixFromContext(ctx)
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 92, "<!doctype html><html lang=\"en\"><head><meta charset=\"UTF-8\"><title>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var61 string
|
||||
templ_7745c5c3_Var61, templ_7745c5c3_Err = templ.JoinStringErrs(title)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 386, Col: 17}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var61))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 93, " - Login</title><link rel=\"icon\" href=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 93, "<!doctype html><html lang=\"en\"><head><meta charset=\"UTF-8\"><title>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var62 string
|
||||
templ_7745c5c3_Var62, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/favicon.ico").ResolveURL(ctx)
|
||||
templ_7745c5c3_Var62, templ_7745c5c3_Err = templ.JoinStringErrs(title)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 387, Col: 58}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 391, Col: 17}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var62)
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var62))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, "\" type=\"image/x-icon\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><link href=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, " - Login</title><link rel=\"icon\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var63 string
|
||||
templ_7745c5c3_Var63, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/css/bootstrap.min.css").ResolveURL(ctx)
|
||||
templ_7745c5c3_Var63, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/favicon.ico").ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 389, Col: 57}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 392, Col: 58}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var63)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 95, "\" rel=\"stylesheet\"><link href=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 95, "\" type=\"image/x-icon\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><link href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var64 string
|
||||
templ_7745c5c3_Var64, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/css/bootstrap-icons.min.css").ResolveURL(ctx)
|
||||
templ_7745c5c3_Var64, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/css/bootstrap.min.css").ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 390, Col: 67}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 394, Col: 57}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var64)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, "\" rel=\"stylesheet\"></head><body class=\"bg-light\"><div class=\"container\"><div class=\"row justify-content-center min-vh-100 align-items-center\"><div class=\"col-md-6 col-lg-4\"><div class=\"card shadow\"><div class=\"card-body p-5\"><div class=\"text-center mb-4\"><i class=\"bi bi-server icon-3x text-primary mb-3\"></i><h4 class=\"card-title\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, "\" rel=\"stylesheet\"><link href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var65 string
|
||||
templ_7745c5c3_Var65, templ_7745c5c3_Err = templ.JoinStringErrs(title)
|
||||
templ_7745c5c3_Var65, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/css/bootstrap-icons.min.css").ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 400, Col: 57}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 395, Col: 67}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var65))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var65)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 97, "</h4><p class=\"text-muted\">Please sign in to continue</p></div>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 97, "\" rel=\"stylesheet\"></head><body class=\"bg-light\"><div class=\"container\"><div class=\"row justify-content-center min-vh-100 align-items-center\"><div class=\"col-md-6 col-lg-4\"><div class=\"card shadow\"><div class=\"card-body p-5\"><div class=\"text-center mb-4\"><i class=\"bi bi-server icon-3x text-primary mb-3\"></i><h4 class=\"card-title\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var66 string
|
||||
templ_7745c5c3_Var66, templ_7745c5c3_Err = templ.JoinStringErrs(title)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 405, Col: 57}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var66))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 98, "</h4><p class=\"text-muted\">Please sign in to continue</p></div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
if errorMessage != "" {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 98, "<div class=\"alert alert-danger\" role=\"alert\"><i class=\"bi bi-exclamation-triangle me-2\"></i> ")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 99, "<div class=\"alert alert-danger\" role=\"alert\"><i class=\"bi bi-exclamation-triangle me-2\"></i> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var66 string
|
||||
templ_7745c5c3_Var66, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage)
|
||||
var templ_7745c5c3_Var67 string
|
||||
templ_7745c5c3_Var67, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 407, Col: 45}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 412, Col: 45}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var66))
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var67))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 99, "</div>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 100, "</div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 100, "<form method=\"POST\" action=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var67 string
|
||||
templ_7745c5c3_Var67, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.SafeURL(prefix + "/login")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 411, Col: 85}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var67)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 101, "\"><input type=\"hidden\" name=\"csrf_token\" value=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 101, "<form method=\"POST\" action=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var68 string
|
||||
templ_7745c5c3_Var68, templ_7745c5c3_Err = templruntime.NewAttributeExpression(csrfToken).Resolve(ctx)
|
||||
templ_7745c5c3_Var68, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.SafeURL(prefix + "/login")).ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 412, Col: 84}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 416, Col: 85}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var68)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 102, "\"><div class=\"mb-3\"><label for=\"username\" class=\"form-label\">Username</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"bi bi-person\"></i></span> <input type=\"text\" class=\"form-control\" id=\"username\" name=\"username\" required></div></div><div class=\"mb-4\"><label for=\"password\" class=\"form-label\">Password</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"bi bi-lock\"></i></span> <input type=\"password\" class=\"form-control\" id=\"password\" name=\"password\" required></div></div><button type=\"submit\" class=\"btn btn-primary w-100\"><i class=\"bi bi-box-arrow-in-right me-2\"></i>Sign In</button></form></div></div></div></div></div><script src=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 102, "\"><input type=\"hidden\" name=\"csrf_token\" value=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var69 string
|
||||
templ_7745c5c3_Var69, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/js/bootstrap.bundle.min.js").ResolveURL(ctx)
|
||||
templ_7745c5c3_Var69, templ_7745c5c3_Err = templruntime.NewAttributeExpression(csrfToken).Resolve(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 443, Col: 63}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 417, Col: 84}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var69)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 103, "\"></script></body></html>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 103, "\"><div class=\"mb-3\"><label for=\"username\" class=\"form-label\">Username</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"bi bi-person\"></i></span> <input type=\"text\" class=\"form-control\" id=\"username\" name=\"username\" required></div></div><div class=\"mb-4\"><label for=\"password\" class=\"form-label\">Password</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"bi bi-lock\"></i></span> <input type=\"password\" class=\"form-control\" id=\"password\" name=\"password\" required></div></div><button type=\"submit\" class=\"btn btn-primary w-100\"><i class=\"bi bi-box-arrow-in-right me-2\"></i>Sign In</button></form></div></div></div></div></div><script src=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var70 string
|
||||
templ_7745c5c3_Var70, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/js/bootstrap.bundle.min.js").ResolveURL(ctx)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 448, Col: 63}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var70)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 104, "\"></script></body></html>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
|
||||
@@ -64,6 +64,7 @@ type AdminOptions struct {
|
||||
debugPort *int
|
||||
cpuProfile *string
|
||||
memProfile *string
|
||||
uploadMaxSizeMB *int
|
||||
|
||||
// workerGrpcListener, when set, is a listener already bound to grpcPort by
|
||||
// the caller. `weed mini` reserves the port this way because the admin
|
||||
@@ -125,6 +126,7 @@ func init() {
|
||||
a.debugPort = cmdAdmin.Flag.Int("debug.port", 6060, "http port for debugging")
|
||||
a.cpuProfile = cmdAdmin.Flag.String("cpuprofile", "", "cpu profile output file")
|
||||
a.memProfile = cmdAdmin.Flag.String("memprofile", "", "memory profile output file")
|
||||
a.uploadMaxSizeMB = cmdAdmin.Flag.Int("uploadMaxSizeMB", dash.DefaultUploadMaxSizeMB, "maximum total size in MB of one file browser upload (0 for no limit)")
|
||||
}
|
||||
|
||||
var cmdAdmin = &Command{
|
||||
@@ -500,6 +502,9 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,
|
||||
s3PublicEndpoint = options.defaultS3PublicEndpoint
|
||||
}
|
||||
adminServer := dash.NewAdminServer(*options.master, *options.filerGroup, nil, dataDir, icebergPort, lancePort, s3PublicEndpoint)
|
||||
if options.uploadMaxSizeMB != nil {
|
||||
adminServer.SetUploadMaxSize(int64(*options.uploadMaxSizeMB) << 20)
|
||||
}
|
||||
|
||||
if err := adminServer.ApplyPluginConfigFromToml(util.GetViper()); err != nil {
|
||||
return fmt.Errorf("apply admin.toml to plugin config: %w", err)
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/sink"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/sink/localsink"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/source"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
@@ -343,3 +346,35 @@ func TestGenProcessFunctionRenameFromExcludedDirBecomesCreate(t *testing.T) {
|
||||
t.Fatalf("unexpected delete/update calls: deletes=%v updates=%v", dataSink.deleteKeys, dataSink.updateKeys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenProcessFunctionLocalSinkStaysInsideTarget(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
target := filepath.Join(root, "backup")
|
||||
dataSink := &localsink.LocalSink{Dir: target}
|
||||
dataSink.SetSourceFiler(&source.FilerSource{})
|
||||
processFn := genProcessFunction("/", target, nil, nil, nil, nil, dataSink, true, false)
|
||||
|
||||
for _, dir := range []string{"/safe", "/../escape"} {
|
||||
err := processFn(&filer_pb.SubscribeMetadataResponse{
|
||||
Directory: dir,
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
NewParentPath: dir,
|
||||
NewEntry: &filer_pb.Entry{
|
||||
Name: "marker.txt",
|
||||
Content: []byte("marker"),
|
||||
Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("processFn %s: %v", dir, err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := os.Stat(filepath.Join(target, "safe", "marker.txt")); err != nil {
|
||||
t.Fatalf("inside file missing: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "escape")); !os.IsNotExist(err) {
|
||||
t.Fatalf("local sink wrote outside %s: %v", target, err)
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
@@ -598,6 +599,7 @@ func initMiniAdminFlags() {
|
||||
miniAdminOptions.allowInsecureBind = cmdMini.Flag.Bool("admin.allowInsecureBind", false, "INSECURE: allow the unauthenticated admin interface to bind to -ip.bind instead of loopback")
|
||||
miniAdminWorkerBindIP = cmdMini.Flag.String("admin.worker.ip", "127.0.0.1", "worker gRPC bind address; expose only with grpc.admin mTLS configured")
|
||||
miniAdminOptions.urlPrefix = cmdMini.Flag.String("admin.urlPrefix", "", "URL path prefix when running the admin UI behind a reverse proxy under a subdirectory (e.g. /seaweedfs)")
|
||||
miniAdminOptions.uploadMaxSizeMB = cmdMini.Flag.Int("admin.uploadMaxSizeMB", dash.DefaultUploadMaxSizeMB, "maximum total size in MB of one file browser upload (0 for no limit)")
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -675,6 +675,7 @@ func (s3opt *S3Options) startLanceServer(s3ApiServer *s3api.S3ApiServer, listenO
|
||||
|
||||
lanceServer := lance.NewServer(s3ApiServer, s3ApiServer)
|
||||
lanceServer.SetCredentialValidator(s3ApiServer)
|
||||
lanceServer.SetSigningKey(s3ApiServer.FilerSigningKey)
|
||||
if s3opt.icebergCredentialRole != nil && *s3opt.icebergCredentialRole != "" {
|
||||
lanceServer.SetCredentialVendor(lanceCredentialVendor{s3ApiServer})
|
||||
}
|
||||
@@ -723,6 +724,7 @@ func (s3opt *S3Options) vectorsRouter(s3ApiServer *s3api.S3ApiServer, grpcDialOp
|
||||
vectorsServer := s3vectors.NewServer(s3ApiServer, s3ApiServer, backend)
|
||||
vectorsServer.SetRegion(s3tables.DefaultRegion)
|
||||
vectorsServer.SetAccountID(s3tables.DefaultAccountID)
|
||||
vectorsServer.SetEncryptionChecker(s3api.GetSSES3KeyManager().IsConfigured)
|
||||
vectorsServer.RegisterRoutes(vectorsRouter)
|
||||
return vectorsRouter
|
||||
}
|
||||
|
||||
@@ -501,6 +501,10 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
|
||||
return filer_pb.ErrEntryNameTooLong
|
||||
}
|
||||
|
||||
if entry.FullPath.HasDotSegment() {
|
||||
return fmt.Errorf("invalid path %s", entry.FullPath)
|
||||
}
|
||||
|
||||
if entry.IsDirectory() {
|
||||
entry.Attr.TtlSec = 0
|
||||
}
|
||||
@@ -794,6 +798,9 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath,
|
||||
}
|
||||
|
||||
func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry, isFromOtherCluster bool) (err error) {
|
||||
if entry.FullPath.HasDotSegment() {
|
||||
return fmt.Errorf("invalid path %s", entry.FullPath)
|
||||
}
|
||||
ctx = f.syncWriteContext(ctx, entry.FullPath)
|
||||
if oldEntry != nil {
|
||||
entry.Attr.Crtime = oldEntry.Attr.Crtime
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
func (f *Filer) CanRename(ctx context.Context, source, target util.FullPath, oldName string) error {
|
||||
sourcePath := source.Child(oldName)
|
||||
if strings.HasPrefix(string(target), string(sourcePath)) {
|
||||
if util.IsEqualOrUnder(string(target), string(sourcePath)) {
|
||||
return fmt.Errorf("mv: can not move directory to a subdirectory of itself")
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
@@ -29,3 +32,39 @@ func TestDetectBucket(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanRenameIntoItself(t *testing.T) {
|
||||
f, store := newTestFilerWithStubStore(t)
|
||||
f.DirBucketsPath = "/buckets"
|
||||
if err := store.InsertEntry(context.Background(), &Entry{FullPath: "/dir/a", Attr: Attr{Mode: os.ModeDir}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
oldName string
|
||||
target string
|
||||
reject bool
|
||||
}{
|
||||
{"into itself", "a", "/dir/a", true},
|
||||
{"into a child", "a", "/dir/a/b", true},
|
||||
{"into a deeper descendant", "a", "/dir/a/b/c", true},
|
||||
{"into a child with a trailing slash", "a", "/dir/a/b/", true},
|
||||
{"old name with a trailing slash into a child", "a/", "/dir/a/b", true},
|
||||
{"into a sibling sharing the name prefix", "a", "/dir/a.d", false},
|
||||
{"into a descendant of that sibling", "a", "/dir/ab/c", false},
|
||||
{"into the parent", "a", "/dir", false},
|
||||
{"into an unrelated directory", "a", "/other", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := f.CanRename(context.Background(), "/dir", util.FullPath(tt.target), tt.oldName)
|
||||
rejected := err != nil && strings.Contains(err.Error(), "subdirectory of itself")
|
||||
if rejected != tt.reject {
|
||||
t.Errorf("CanRename(/dir, %q, %q) = %v, want rejected=%v", tt.target, tt.oldName, err, tt.reject)
|
||||
}
|
||||
if !tt.reject && err != nil {
|
||||
t.Errorf("CanRename(/dir, %q, %q) = %v, want nil", tt.target, tt.oldName, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
@@ -167,9 +168,7 @@ func (store *LevelDB3Store) findDB(fullpath weed_util.FullPath, isForChildren bo
|
||||
shortPath = weed_util.FullPath(bucketAndObjectKey[t:])
|
||||
}
|
||||
|
||||
// Dot-prefixed entries directly under /buckets (e.g. .system) are internal
|
||||
// folders, not S3 buckets; keep them in the default DB by full path.
|
||||
if strings.HasPrefix(bucket, ".") {
|
||||
if !isBucketFolder(bucket) {
|
||||
store.dbsLock.RUnlock()
|
||||
return defaultDB, DEFAULT, fullpath, nil
|
||||
}
|
||||
@@ -207,6 +206,14 @@ func (store *LevelDB3Store) emptyBucketDB() (*leveldb.DB, error) {
|
||||
return store.emptyDB, nil
|
||||
}
|
||||
|
||||
// Dot-prefixed entries directly under /buckets (e.g. .system) are internal
|
||||
// folders, not S3 buckets; they stay in the default DB by full path. A bucket
|
||||
// also names a single folder under store.dir, so it cannot hold a separator;
|
||||
// "\" is an ordinary name character on Unix and only rejected where it separates.
|
||||
func isBucketFolder(bucket string) bool {
|
||||
return !strings.HasPrefix(bucket, ".") && filepath.IsLocal(bucket) && !strings.ContainsAny(bucket, "/"+string(os.PathSeparator))
|
||||
}
|
||||
|
||||
func (store *LevelDB3Store) createDB(bucket string) (*leveldb.DB, error) {
|
||||
|
||||
store.dbsLock.Lock()
|
||||
|
||||
@@ -9,14 +9,18 @@ import (
|
||||
var _ filer.BucketAware = (*LevelDB3Store)(nil)
|
||||
|
||||
func (store *LevelDB3Store) OnBucketCreation(bucket string) {
|
||||
if !isBucketFolder(bucket) {
|
||||
return
|
||||
}
|
||||
store.createDB(bucket)
|
||||
}
|
||||
|
||||
func (store *LevelDB3Store) OnBucketDeletion(bucket string) {
|
||||
store.closeDB(bucket)
|
||||
if bucket != "" { // just to make sure
|
||||
os.RemoveAll(store.dir + "/" + bucket)
|
||||
if !isBucketFolder(bucket) || bucket == DEFAULT {
|
||||
return
|
||||
}
|
||||
store.closeDB(bucket)
|
||||
os.RemoveAll(store.dir + "/" + bucket)
|
||||
}
|
||||
|
||||
func (store *LevelDB3Store) CanDropWholeBucket() bool {
|
||||
|
||||
@@ -3,6 +3,8 @@ package leveldb
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -207,3 +209,44 @@ func TestListDirectoryPrefixedEntries_StartBelowPrefix(t *testing.T) {
|
||||
t.Fatalf("want [item-b.versions], got %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketEventsStayInsideStoreDir(t *testing.T) {
|
||||
parent := t.TempDir()
|
||||
dir := filepath.Join(parent, "store")
|
||||
store := &LevelDB3Store{}
|
||||
if err := store.initialize(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
|
||||
sentinel := filepath.Join(parent, "keep")
|
||||
if err := os.WriteFile(sentinel, []byte("keep"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Checked after each create, before the matching delete could remove the
|
||||
// evidence of a db opened in the wrong place.
|
||||
assertNothingEscaped := func(bucket string) {
|
||||
t.Helper()
|
||||
for _, p := range []string{filepath.Join(parent, "outside"), filepath.Join(parent, "CURRENT"), filepath.Join(dir, "x", "y")} {
|
||||
if _, err := os.Stat(p); !os.IsNotExist(err) {
|
||||
t.Fatalf("bucket %q event created %s: %v", bucket, p, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
store.OnBucketCreation("x")
|
||||
for _, bucket := range []string{"..", ".", "../outside", "", DEFAULT, "x/../" + DEFAULT, "x/y"} {
|
||||
store.OnBucketCreation(bucket)
|
||||
assertNothingEscaped(bucket)
|
||||
store.OnBucketDeletion(bucket)
|
||||
}
|
||||
if _, err := os.Stat(sentinel); err != nil {
|
||||
t.Fatalf("bucket events removed %s: %v", sentinel, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, DEFAULT)); err != nil {
|
||||
t.Fatalf("default db folder missing: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
if err := store.InsertEntry(ctx, &filer.Entry{FullPath: "/a/b"}); err != nil {
|
||||
t.Fatalf("default db unusable after bucket events: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
@@ -94,7 +95,7 @@ func (cm *ChunkManifest) DeleteChunks(masterFn GetMasterFn, usePublicUrl bool, g
|
||||
return nil
|
||||
}
|
||||
|
||||
func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (written int64, e error) {
|
||||
func readChunkNeedle(fileUrl string, w io.Writer, offset int64, size int64, jwt string) (written int64, e error) {
|
||||
req, err := http.NewRequest(http.MethodGet, fileUrl, nil)
|
||||
if err != nil {
|
||||
return written, err
|
||||
@@ -125,7 +126,11 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (wri
|
||||
return written, util_http.StatusError(fileUrl, resp)
|
||||
|
||||
}
|
||||
return io.Copy(w, resp.Body)
|
||||
written, e = io.CopyN(w, resp.Body, size)
|
||||
if e == io.EOF {
|
||||
e = fmt.Errorf("%s: %d of %d bytes: %w", fileUrl, written, size, io.ErrUnexpectedEOF)
|
||||
}
|
||||
return written, e
|
||||
}
|
||||
|
||||
func NewChunkedFileReader(chunkList []*ChunkInfo, master pb.ServerAddress, grpcDialOption grpc.DialOption) *ChunkedFileReader {
|
||||
@@ -175,26 +180,69 @@ func (cf *ChunkedFileReader) WriteTo(w io.Writer) (n int64, err error) {
|
||||
return n, ErrInvalidRange
|
||||
}
|
||||
for ; chunkIndex < len(cf.chunkList); chunkIndex++ {
|
||||
ci := cf.chunkList[chunkIndex]
|
||||
// if we need read date from local volume server first?
|
||||
fileUrl, jwt, lookupError := LookupFileId(func(_ context.Context) pb.ServerAddress {
|
||||
return cf.master
|
||||
}, cf.grpcDialOption, ci.Fid)
|
||||
if lookupError != nil {
|
||||
return n, lookupError
|
||||
}
|
||||
if wn, e := readChunkNeedle(fileUrl, w, chunkStartOffset, jwt); e != nil {
|
||||
wn, e := cf.readChunk(cf.chunkList[chunkIndex], w, chunkStartOffset)
|
||||
n += wn
|
||||
cf.pos += wn
|
||||
if e != nil {
|
||||
return n, e
|
||||
} else {
|
||||
n += wn
|
||||
cf.pos += wn
|
||||
}
|
||||
|
||||
chunkStartOffset = 0
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// readChunk reads the chunk from the first replica that serves it. A failure
|
||||
// drops the volume's cached locations, and once every cached replica has
|
||||
// failed the master is asked again. A replica that already sent part of the
|
||||
// chunk is not continued elsewhere: a volume server streams a needle before
|
||||
// its checksum is verified, so those bytes may be the corrupt ones.
|
||||
func (cf *ChunkedFileReader) readChunk(ci *ChunkInfo, w io.Writer, offset int64) (written int64, err error) {
|
||||
masterFn := func(_ context.Context) pb.ServerAddress {
|
||||
return cf.master
|
||||
}
|
||||
vid, _, _ := strings.Cut(ci.Fid, ",")
|
||||
dst := &writeErrorRecorder{Writer: w}
|
||||
tried := make(map[string]bool)
|
||||
for range 2 {
|
||||
fileUrls, jwt, lookupError := lookupFileIdUrls(masterFn, cf.grpcDialOption, ci.Fid)
|
||||
if lookupError != nil {
|
||||
if err == nil {
|
||||
err = lookupError
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
for _, fileUrl := range fileUrls {
|
||||
if tried[fileUrl] {
|
||||
continue
|
||||
}
|
||||
tried[fileUrl] = true
|
||||
written, err = readChunkNeedle(fileUrl, dst, offset, ci.Size-offset, jwt)
|
||||
if err == nil || dst.err != nil {
|
||||
return written, err
|
||||
}
|
||||
glog.V(0).Infof("read chunk %s from %s: %v", ci.Fid, fileUrl, err)
|
||||
InvalidateVolumeIdLocationCache(vid)
|
||||
if written > 0 {
|
||||
return written, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
type writeErrorRecorder struct {
|
||||
io.Writer
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *writeErrorRecorder) Write(p []byte) (int, error) {
|
||||
n, err := r.Writer.Write(p)
|
||||
if err != nil {
|
||||
r.err = err
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (cf *ChunkedFileReader) ReadAt(p []byte, off int64) (n int, err error) {
|
||||
cf.Seek(off, 0)
|
||||
return cf.Read(p)
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
package operation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
type fakeVolumeLocationServer struct {
|
||||
master_pb.UnimplementedSeaweedServer
|
||||
mu sync.Mutex
|
||||
calls int
|
||||
urls []string
|
||||
}
|
||||
|
||||
func (s *fakeVolumeLocationServer) LookupVolume(_ context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.calls++
|
||||
resp := &master_pb.LookupVolumeResponse{}
|
||||
for _, vid := range req.VolumeOrFileIds {
|
||||
var locs []*master_pb.Location
|
||||
for _, u := range s.urls {
|
||||
locs = append(locs, &master_pb.Location{Url: u})
|
||||
}
|
||||
resp.VolumeIdLocations = append(resp.VolumeIdLocations, &master_pb.LookupVolumeResponse_VolumeIdLocation{
|
||||
VolumeOrFileId: vid,
|
||||
Locations: locs,
|
||||
})
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *fakeVolumeLocationServer) lookups() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.calls
|
||||
}
|
||||
|
||||
type chunkReplica struct {
|
||||
*httptest.Server
|
||||
mu sync.Mutex
|
||||
ranges []string
|
||||
}
|
||||
|
||||
func (c *chunkReplica) host() string {
|
||||
return strings.TrimPrefix(c.URL, "http://")
|
||||
}
|
||||
|
||||
func (c *chunkReplica) requests() []string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return append([]string(nil), c.ranges...)
|
||||
}
|
||||
|
||||
// newChunkReplica serves data; with cutAfter >= 0 it announces the full length
|
||||
// but drops the connection after cutAfter bytes.
|
||||
func newChunkReplica(t *testing.T, data []byte, cutAfter int) *chunkReplica {
|
||||
c := &chunkReplica{}
|
||||
c.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
c.mu.Lock()
|
||||
c.ranges = append(c.ranges, r.Header.Get("Range"))
|
||||
c.mu.Unlock()
|
||||
if cutAfter < 0 {
|
||||
http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(data))
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(data[:cutAfter])
|
||||
w.(http.Flusher).Flush()
|
||||
conn, _, err := w.(http.Hijacker).Hijack()
|
||||
if err == nil {
|
||||
conn.Close()
|
||||
}
|
||||
}))
|
||||
t.Cleanup(c.Close)
|
||||
return c
|
||||
}
|
||||
|
||||
func deadAddress(t *testing.T) string {
|
||||
lis, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
addr := lis.Addr().String()
|
||||
lis.Close()
|
||||
return addr
|
||||
}
|
||||
|
||||
func testChunkData(size int) []byte {
|
||||
data := make([]byte, size)
|
||||
for i := range data {
|
||||
data[i] = byte(i * 7)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func TestChunkedFileReaderTriesTheNextReplica(t *testing.T) {
|
||||
const vid = "91201"
|
||||
data := testChunkData(4096)
|
||||
healthy := newChunkReplica(t, data, -1)
|
||||
fake := &fakeVolumeLocationServer{urls: []string{healthy.host()}}
|
||||
master := startFakeMasterServer(t, fake)
|
||||
dial := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
|
||||
vc.Set(vid, []Location{{Url: deadAddress(t)}, {Url: healthy.host(), DataInRemote: true}}, time.Minute)
|
||||
t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) })
|
||||
|
||||
chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}}
|
||||
reader := NewChunkedFileReader(chunks, master, dial)
|
||||
reader.Seek(1000, io.SeekStart)
|
||||
got, err := io.ReadAll(reader)
|
||||
reader.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, data[1000:]) {
|
||||
t.Fatalf("read %d bytes, want the last %d bytes of the chunk", len(got), len(data)-1000)
|
||||
}
|
||||
if r := healthy.requests(); len(r) != 1 || r[0] != "bytes=1000-" {
|
||||
t.Fatalf("healthy replica requests = %q, want one from byte 1000", r)
|
||||
}
|
||||
if _, err := vc.Get(vid); err == nil {
|
||||
t.Fatalf("cached locations kept after a replica failed")
|
||||
}
|
||||
if fake.lookups() != 0 {
|
||||
t.Fatalf("master lookups = %d, want 0", fake.lookups())
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkedFileReaderDoesNotContinueAPartlySentChunk(t *testing.T) {
|
||||
const vid = "91205"
|
||||
data := testChunkData(4096)
|
||||
broken := newChunkReplica(t, data, 1000)
|
||||
healthy := newChunkReplica(t, data, -1)
|
||||
fake := &fakeVolumeLocationServer{urls: []string{healthy.host()}}
|
||||
master := startFakeMasterServer(t, fake)
|
||||
dial := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
|
||||
vc.Set(vid, []Location{{Url: broken.host()}, {Url: healthy.host(), DataInRemote: true}}, time.Minute)
|
||||
t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) })
|
||||
|
||||
chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}}
|
||||
reader := NewChunkedFileReader(chunks, master, dial)
|
||||
_, err := io.ReadAll(reader)
|
||||
reader.Close()
|
||||
if err == nil {
|
||||
t.Fatalf("read of a chunk cut off mid-body succeeded")
|
||||
}
|
||||
if r := healthy.requests(); len(r) != 0 {
|
||||
t.Fatalf("healthy replica requests = %q, want none", r)
|
||||
}
|
||||
|
||||
reader = NewChunkedFileReader(chunks, master, dial)
|
||||
got, err := io.ReadAll(reader)
|
||||
reader.Close()
|
||||
if err != nil || !bytes.Equal(got, data) {
|
||||
t.Fatalf("second read: %d bytes, err %v", len(got), err)
|
||||
}
|
||||
if fake.lookups() != 1 || len(broken.requests()) != 1 {
|
||||
t.Fatalf("second read: %d master lookups, %d requests to the broken replica; want 1 and 1", fake.lookups(), len(broken.requests()))
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkedFileReaderAsksMasterWhenCachedReplicasFail(t *testing.T) {
|
||||
const vid = "91202"
|
||||
first, second := testChunkData(3000), testChunkData(2000)
|
||||
healthy := newChunkReplica(t, second, -1)
|
||||
fake := &fakeVolumeLocationServer{urls: []string{healthy.host()}}
|
||||
master := startFakeMasterServer(t, fake)
|
||||
dial := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
|
||||
firstReplica := newChunkReplica(t, first, -1)
|
||||
const otherVid = "91203"
|
||||
vc.Set(otherVid, []Location{{Url: firstReplica.host()}}, time.Minute)
|
||||
vc.Set(vid, []Location{{Url: deadAddress(t)}}, time.Minute)
|
||||
t.Cleanup(func() {
|
||||
InvalidateVolumeIdLocationCache(vid)
|
||||
InvalidateVolumeIdLocationCache(otherVid)
|
||||
})
|
||||
|
||||
chunks := []*ChunkInfo{
|
||||
{Fid: otherVid + ",01637037d6", Offset: 0, Size: int64(len(first))},
|
||||
{Fid: vid + ",02637037d6", Offset: int64(len(first)), Size: int64(len(second))},
|
||||
}
|
||||
reader := NewChunkedFileReader(chunks, master, dial)
|
||||
defer reader.Close()
|
||||
got, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
if want := append(append([]byte(nil), first...), second...); !bytes.Equal(got, want) {
|
||||
t.Fatalf("read %d bytes, want %d", len(got), len(want))
|
||||
}
|
||||
if fake.lookups() != 1 {
|
||||
t.Fatalf("master lookups = %d, want 1", fake.lookups())
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkedFileReaderTreatsAShortBodyAsFailed(t *testing.T) {
|
||||
const vid = "91206"
|
||||
data := testChunkData(2048)
|
||||
empty := newChunkReplica(t, nil, -1)
|
||||
healthy := newChunkReplica(t, data, -1)
|
||||
vc.Set(vid, []Location{{Url: empty.host()}, {Url: healthy.host(), DataInRemote: true}}, time.Minute)
|
||||
t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) })
|
||||
|
||||
chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}}
|
||||
reader := NewChunkedFileReader(chunks, pb.ServerAddress("127.0.0.1:1"), grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
defer reader.Close()
|
||||
got, err := io.ReadAll(reader)
|
||||
if err != nil || !bytes.Equal(got, data) {
|
||||
t.Fatalf("read: %d bytes, err %v; want %d bytes", len(got), err, len(data))
|
||||
}
|
||||
if len(empty.requests()) != 1 || len(healthy.requests()) != 1 {
|
||||
t.Fatalf("requests: empty %d, healthy %d; want 1 and 1", len(empty.requests()), len(healthy.requests()))
|
||||
}
|
||||
}
|
||||
|
||||
type failingWriter struct{}
|
||||
|
||||
func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("client gone") }
|
||||
|
||||
func TestChunkedFileReaderStopsWhenTheClientGoesAway(t *testing.T) {
|
||||
const vid = "91204"
|
||||
data := testChunkData(1024)
|
||||
a, b := newChunkReplica(t, data, -1), newChunkReplica(t, data, -1)
|
||||
vc.Set(vid, []Location{{Url: a.host()}, {Url: b.host()}}, time.Minute)
|
||||
t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) })
|
||||
|
||||
chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}}
|
||||
reader := NewChunkedFileReader(chunks, pb.ServerAddress("127.0.0.1:1"), grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if _, err := reader.WriteTo(failingWriter{}); err == nil || err.Error() != "client gone" {
|
||||
t.Fatalf("WriteTo error = %v, want the writer's error", err)
|
||||
}
|
||||
if n := len(a.requests()) + len(b.requests()); n != 1 {
|
||||
t.Fatalf("replica requests = %d, want 1", n)
|
||||
}
|
||||
if _, err := vc.Get(vid); err != nil {
|
||||
t.Fatalf("a client write error dropped the cached locations: %v", err)
|
||||
}
|
||||
}
|
||||
+22
-12
@@ -51,31 +51,41 @@ var (
|
||||
// authorization the master stamped on the volume; pass it through to the
|
||||
// volume server on the read request.
|
||||
func LookupFileId(masterFn GetMasterFn, grpcDialOption grpc.DialOption, fileId string) (fullUrl string, jwt string, err error) {
|
||||
fullUrls, jwt, err := lookupFileIdUrls(masterFn, grpcDialOption, fileId)
|
||||
if err != nil {
|
||||
return "", jwt, err
|
||||
}
|
||||
return fullUrls[0], jwt, nil
|
||||
}
|
||||
|
||||
// lookupFileIdUrls returns the read URLs of every replica of fileId, local
|
||||
// replicas first, each group in random order.
|
||||
func lookupFileIdUrls(masterFn GetMasterFn, grpcDialOption grpc.DialOption, fileId string) (fullUrls []string, jwt string, err error) {
|
||||
parts := strings.Split(fileId, ",")
|
||||
if len(parts) != 2 {
|
||||
return "", jwt, errors.New("Invalid fileId " + fileId)
|
||||
return nil, jwt, errors.New("Invalid fileId " + fileId)
|
||||
}
|
||||
lookup, lookupError := LookupVolumeId(masterFn, grpcDialOption, parts[0])
|
||||
if lookupError != nil {
|
||||
return "", jwt, lookupError
|
||||
return nil, jwt, lookupError
|
||||
}
|
||||
if len(lookup.Locations) == 0 {
|
||||
return "", jwt, errors.New("File Not Found")
|
||||
return nil, jwt, errors.New("File Not Found")
|
||||
}
|
||||
|
||||
localUrls := make([]string, 0, len(lookup.Locations))
|
||||
var localUrls, remoteUrls []string
|
||||
for _, loc := range lookup.Locations {
|
||||
if !loc.DataInRemote {
|
||||
localUrls = append(localUrls, loc.Url)
|
||||
}
|
||||
}
|
||||
if len(localUrls) == 0 {
|
||||
for _, loc := range lookup.Locations {
|
||||
localUrls = append(localUrls, loc.Url)
|
||||
fullUrl := "http://" + loc.Url + "/" + fileId
|
||||
if loc.DataInRemote {
|
||||
remoteUrls = append(remoteUrls, fullUrl)
|
||||
} else {
|
||||
localUrls = append(localUrls, fullUrl)
|
||||
}
|
||||
}
|
||||
rand.Shuffle(len(localUrls), func(i, j int) { localUrls[i], localUrls[j] = localUrls[j], localUrls[i] })
|
||||
rand.Shuffle(len(remoteUrls), func(i, j int) { remoteUrls[i], remoteUrls[j] = remoteUrls[j], remoteUrls[i] })
|
||||
|
||||
return "http://" + localUrls[rand.IntN(len(localUrls))] + "/" + fileId, lookup.Jwt, nil
|
||||
return append(localUrls, remoteUrls...), lookup.Jwt, nil
|
||||
}
|
||||
|
||||
func LookupVolumeId(masterFn GetMasterFn, grpcDialOption grpc.DialOption, vid string) (*LookupResult, error) {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
@@ -118,6 +119,54 @@ func TestCreateNeedleFromRequest(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func TestUploadKeepsMarginalGzipUncompressed(t *testing.T) {
|
||||
data := make([]byte, 4300)
|
||||
rand.New(rand.NewSource(1)).Read(data[:4000])
|
||||
gzipped, _ := util.GzipData(data)
|
||||
if len(gzipped) >= len(data) || len(gzipped)*10 < len(data)*9 {
|
||||
t.Fatalf("gzip size %d of %d is not a marginal saving", len(gzipped), len(data))
|
||||
}
|
||||
|
||||
mockClient := &MockClient{}
|
||||
var stored *needle.Needle
|
||||
mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) {
|
||||
assert.Equal(t, nil, err, "upload: %v", err)
|
||||
stored = n
|
||||
}
|
||||
uploadOption := &UploadOption{
|
||||
UploadUrl: "http://localhost:8080/389,0f084d17353afda0",
|
||||
Filename: "doc.pdf",
|
||||
MimeType: "application/pdf",
|
||||
}
|
||||
newUploader(mockClient).Upload(context.Background(), bytes.NewReader(data), uploadOption)
|
||||
if stored == nil {
|
||||
t.Fatal("no needle reached the volume server")
|
||||
}
|
||||
assert.False(t, stored.IsCompressed(), "gzip saved less than 10%, data should stay uncompressed")
|
||||
assert.Equal(t, data, stored.Data)
|
||||
}
|
||||
|
||||
func TestReplicationKeepsTheSourceMime(t *testing.T) {
|
||||
for _, mimeType := range []string{"", "application/octet-stream", "Text/Plain; charset=UTF-8"} {
|
||||
mockClient := &MockClient{}
|
||||
var stored *needle.Needle
|
||||
mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) {
|
||||
assert.Equal(t, nil, err, "upload: %v", err)
|
||||
stored = n
|
||||
}
|
||||
newUploader(mockClient).UploadData(context.Background(), []byte(textContent), &UploadOption{
|
||||
UploadUrl: "http://localhost:8080/389,0f084d17353afda0?type=replicate",
|
||||
Filename: "report.html",
|
||||
MimeType: mimeType,
|
||||
IsReplication: true,
|
||||
})
|
||||
if stored == nil {
|
||||
t.Fatal("no needle reached the volume server")
|
||||
}
|
||||
assert.Equal(t, mimeType, string(stored.Mime))
|
||||
}
|
||||
}
|
||||
|
||||
var textContent = `Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
@@ -390,7 +390,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
|
||||
option.MimeType = ""
|
||||
}
|
||||
}
|
||||
if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(option.Filename), option.MimeType); iAmSure && shouldBeCompressed {
|
||||
if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Ext(option.Filename), option.MimeType); iAmSure && shouldBeCompressed {
|
||||
shouldGzipNow = true
|
||||
} else if !iAmSure && option.MimeType == "" && len(data) > 16*1024 {
|
||||
var compressed []byte
|
||||
@@ -412,7 +412,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
|
||||
compressed, compressErr := util.GzipData(data)
|
||||
// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
|
||||
if compressErr == nil {
|
||||
if len(compressed) < len(data) {
|
||||
if len(compressed)*10 < len(data)*9 {
|
||||
data = compressed
|
||||
contentIsGzipped = true
|
||||
}
|
||||
@@ -474,6 +474,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
|
||||
Filename: option.Filename,
|
||||
Cipher: false,
|
||||
IsInputCompressed: contentIsGzipped,
|
||||
IsReplication: option.IsReplication,
|
||||
MimeType: option.MimeType,
|
||||
PairMap: option.PairMap,
|
||||
Jwt: option.Jwt,
|
||||
@@ -510,7 +511,7 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction
|
||||
// properly handling non-ASCII characters and special characters
|
||||
h.Set("Content-Disposition", mime.FormatMediaType("form-data", map[string]string{"name": "file", "filename": option.Filename}))
|
||||
h.Set("Idempotency-Key", option.UploadUrl)
|
||||
if option.MimeType == "" {
|
||||
if option.MimeType == "" && !option.IsReplication {
|
||||
option.MimeType = mime.TypeByExtension(strings.ToLower(filepath.Ext(option.Filename)))
|
||||
}
|
||||
if option.MimeType != "" {
|
||||
|
||||
@@ -3,6 +3,7 @@ package localsink
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -39,6 +40,34 @@ func (localsink *LocalSink) isMultiPartEntry(key string) bool {
|
||||
return strings.HasSuffix(key, ".part") && strings.Contains(key, "/"+s3_constants.MultipartUploadsFolder+"/")
|
||||
}
|
||||
|
||||
func (localsink *LocalSink) isInsideDir(key string) (string, bool) {
|
||||
rel, err := filepath.Rel(filepath.Clean(localsink.Dir), filepath.Clean(key))
|
||||
if err == nil && filepath.IsLocal(rel) {
|
||||
return rel, true
|
||||
}
|
||||
glog.Warningf("local sink: skip %s outside %s", key, localsink.Dir)
|
||||
return "", false
|
||||
}
|
||||
|
||||
// resolveInsideDir resolves symlinks in the parent of rel while it stays inside the sink directory.
|
||||
func (localsink *LocalSink) resolveInsideDir(rel string) string {
|
||||
base, err := filepath.EvalSymlinks(localsink.Dir)
|
||||
if err != nil {
|
||||
return rel
|
||||
}
|
||||
dir, rest := filepath.Dir(rel), filepath.Base(rel)
|
||||
for dir != "." {
|
||||
if resolved, err := filepath.EvalSymlinks(filepath.Join(base, dir)); err == nil {
|
||||
if inside, err := filepath.Rel(base, resolved); err == nil && filepath.IsLocal(inside) {
|
||||
return filepath.Join(inside, rest)
|
||||
}
|
||||
return rel
|
||||
}
|
||||
dir, rest = filepath.Dir(dir), filepath.Join(filepath.Base(dir), rest)
|
||||
}
|
||||
return rel
|
||||
}
|
||||
|
||||
func (localsink *LocalSink) initialize(dir string, isIncremental bool) error {
|
||||
localsink.Dir = dir
|
||||
localsink.isIncremental = isIncremental
|
||||
@@ -69,8 +98,17 @@ func (localsink *LocalSink) DeleteEntry(key string, isDirectory, deleteIncludeCh
|
||||
if localsink.isMultiPartEntry(key) {
|
||||
return nil
|
||||
}
|
||||
rel, ok := localsink.isInsideDir(key)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
glog.V(4).Infof("Delete Entry key: %s", key)
|
||||
if err := os.Remove(key); err != nil {
|
||||
root, err := os.OpenRoot(localsink.Dir)
|
||||
if err == nil {
|
||||
err = root.Remove(localsink.resolveInsideDir(rel))
|
||||
root.Close()
|
||||
}
|
||||
if err != nil {
|
||||
glog.V(0).Infof("remove entry key %s: %s", key, err)
|
||||
}
|
||||
return nil
|
||||
@@ -81,6 +119,10 @@ func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signa
|
||||
if entry.IsDirectory || localsink.isMultiPartEntry(key) {
|
||||
return nil
|
||||
}
|
||||
rel, ok := localsink.isInsideDir(key)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
glog.V(4).Infof("Create Entry key: %s", key)
|
||||
|
||||
totalSize := filer.FileSize(entry)
|
||||
@@ -89,35 +131,37 @@ func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signa
|
||||
return fmt.Errorf("resolve chunks of %s: %w", key, err)
|
||||
}
|
||||
|
||||
dir := filepath.Dir(key)
|
||||
|
||||
if _, err := os.Stat(dir); os.IsNotExist(err) {
|
||||
glog.V(4).Infof("Create Directory key: %s", dir)
|
||||
if err = os.MkdirAll(dir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(localsink.Dir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
root, err := os.OpenRoot(localsink.Dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer root.Close()
|
||||
|
||||
if entry.IsDirectory {
|
||||
return os.Mkdir(key, os.FileMode(entry.Attributes.FileMode))
|
||||
rel = localsink.resolveInsideDir(rel)
|
||||
dir := filepath.Dir(rel)
|
||||
if err := root.MkdirAll(dir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mode := os.FileMode(entry.Attributes.FileMode)
|
||||
shortFileName := util.ToShortFileName(key)
|
||||
shortFileName := util.ToShortFileName(rel)
|
||||
|
||||
// Write to a temp file in the same directory, then atomically rename
|
||||
// on success. This prevents leaving a truncated/empty file if
|
||||
// decryption or chunk copy fails.
|
||||
tmpFile, err := os.CreateTemp(dir, ".seaweedfs-tmp-*")
|
||||
tmpName := filepath.Join(dir, fmt.Sprintf(".seaweedfs-tmp-%d", rand.Uint64()))
|
||||
tmpFile, err := root.OpenFile(tmpName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpName := tmpFile.Name()
|
||||
defer func() {
|
||||
// Clean up temp file on any error (rename removes it on success)
|
||||
if tmpFile != nil {
|
||||
tmpFile.Close()
|
||||
os.Remove(tmpName)
|
||||
root.Remove(tmpName)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -151,8 +195,8 @@ func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signa
|
||||
tmpFile = nil // prevent deferred cleanup
|
||||
|
||||
// Atomic rename into final destination
|
||||
if err := os.Rename(tmpName, shortFileName); err != nil {
|
||||
os.Remove(tmpName)
|
||||
if err := root.Rename(tmpName, shortFileName); err != nil {
|
||||
root.Remove(tmpName)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -164,6 +208,9 @@ func (localsink *LocalSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, ne
|
||||
if localsink.isMultiPartEntry(key) {
|
||||
return true, nil
|
||||
}
|
||||
if _, ok := localsink.isInsideDir(key); !ok {
|
||||
return true, nil
|
||||
}
|
||||
glog.V(4).Infof("Update Entry key: %s", key)
|
||||
// do delete and create
|
||||
foundExistingEntry = util.FileExists(key)
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
package localsink
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/source"
|
||||
)
|
||||
|
||||
func TestLocalSinkStaysInsideDir(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
dir := filepath.Join(root, "backup")
|
||||
sink := &LocalSink{}
|
||||
sink.initialize(dir, false)
|
||||
sink.SetSourceFiler(&source.FilerSource{})
|
||||
|
||||
entry := func() *filer_pb.Entry {
|
||||
return &filer_pb.Entry{
|
||||
Name: "marker.txt",
|
||||
Content: []byte("marker"),
|
||||
Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6},
|
||||
}
|
||||
}
|
||||
|
||||
inside := filepath.Join(dir, "safe", "marker.txt")
|
||||
if err := sink.CreateEntry(inside, entry(), nil); err != nil {
|
||||
t.Fatalf("create inside: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(inside); err != nil {
|
||||
t.Fatalf("inside file missing: %v", err)
|
||||
}
|
||||
|
||||
outside := filepath.Join(dir, "..", "escape", "marker.txt")
|
||||
if err := sink.CreateEntry(outside, entry(), nil); err != nil {
|
||||
t.Fatalf("create outside: %v", err)
|
||||
}
|
||||
if _, err := sink.UpdateEntry(outside, nil, "", entry(), false, nil); err != nil {
|
||||
t.Fatalf("update outside: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "escape")); !os.IsNotExist(err) {
|
||||
t.Fatalf("sink wrote outside %s: %v", dir, err)
|
||||
}
|
||||
|
||||
victim := filepath.Join(root, "victim.txt")
|
||||
if err := os.WriteFile(victim, []byte("keep"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := sink.DeleteEntry(filepath.Join(dir, "..", "victim.txt"), false, false, nil); err != nil {
|
||||
t.Fatalf("delete outside: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(victim); err != nil {
|
||||
t.Fatalf("sink deleted a file outside %s: %v", dir, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalSinkDoesNotFollowSymlinksOutOfDir(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
dir := filepath.Join(root, "backup")
|
||||
outside := filepath.Join(root, "outside")
|
||||
for _, d := range []string{dir, outside} {
|
||||
if err := os.MkdirAll(d, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.Symlink(outside, filepath.Join(dir, "link")); err != nil {
|
||||
t.Skipf("symlink: %v", err)
|
||||
}
|
||||
sink := &LocalSink{}
|
||||
sink.initialize(dir, false)
|
||||
sink.SetSourceFiler(&source.FilerSource{})
|
||||
|
||||
entry := &filer_pb.Entry{
|
||||
Name: "marker.txt",
|
||||
Content: []byte("marker"),
|
||||
Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6},
|
||||
}
|
||||
if err := sink.CreateEntry(filepath.Join(dir, "link", "marker.txt"), entry, nil); err == nil {
|
||||
t.Fatalf("create through a symlink out of %s succeeded", dir)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(outside, "marker.txt")); !os.IsNotExist(err) {
|
||||
t.Fatalf("sink wrote through a symlink out of %s: %v", dir, err)
|
||||
}
|
||||
|
||||
victim := filepath.Join(outside, "victim.txt")
|
||||
if err := os.WriteFile(victim, []byte("keep"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := sink.DeleteEntry(filepath.Join(dir, "link", "victim.txt"), false, false, nil); err != nil {
|
||||
t.Fatalf("delete through link: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(victim); err != nil {
|
||||
t.Fatalf("sink deleted through a symlink out of %s: %v", dir, err)
|
||||
}
|
||||
|
||||
if err := os.Mkdir(filepath.Join(dir, "real"), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink("real", filepath.Join(dir, "inner")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := sink.CreateEntry(filepath.Join(dir, "inner", "marker.txt"), entry, nil); err != nil {
|
||||
t.Fatalf("create through a symlink inside %s: %v", dir, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "real", "marker.txt")); err != nil {
|
||||
t.Fatalf("inside symlink target missing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalSinkFollowsAbsoluteSymlinksInsideDir(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "backup")
|
||||
target := filepath.Join(dir, "real")
|
||||
if err := os.MkdirAll(target, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(target, filepath.Join(dir, "abs")); err != nil {
|
||||
t.Skipf("symlink: %v", err)
|
||||
}
|
||||
sink := &LocalSink{}
|
||||
sink.initialize(dir, false)
|
||||
sink.SetSourceFiler(&source.FilerSource{})
|
||||
|
||||
entry := &filer_pb.Entry{
|
||||
Name: "marker.txt",
|
||||
Content: []byte("marker"),
|
||||
Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6},
|
||||
}
|
||||
for _, name := range []string{"marker.txt", filepath.Join("sub", "marker.txt")} {
|
||||
if err := sink.CreateEntry(filepath.Join(dir, "abs", name), entry, nil); err != nil {
|
||||
t.Fatalf("create %s through an absolute symlink inside %s: %v", name, dir, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(target, name)); err != nil {
|
||||
t.Fatalf("absolute symlink target %s missing: %v", name, err)
|
||||
}
|
||||
if err := sink.DeleteEntry(filepath.Join(dir, "abs", name), false, false, nil); err != nil {
|
||||
t.Fatalf("delete %s through an absolute symlink: %v", name, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(target, name)); !os.IsNotExist(err) {
|
||||
t.Fatalf("delete through an absolute symlink left %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(dir, "abs")); err != nil {
|
||||
t.Fatalf("symlink itself removed: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -74,22 +75,19 @@ func streamHashRequestBody(r *http.Request, sizeLimit int64) (string, error) {
|
||||
return emptySHA256, nil
|
||||
}
|
||||
|
||||
limitedReader := io.LimitReader(r.Body, sizeLimit)
|
||||
hasher := sha256.New()
|
||||
var bodyBuffer bytes.Buffer
|
||||
|
||||
// Use io.Copy with an io.MultiWriter to hash and buffer the body simultaneously.
|
||||
if _, err := io.Copy(io.MultiWriter(hasher, &bodyBuffer), limitedReader); err != nil {
|
||||
body, err := readRequestBody(r, sizeLimit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
r.Body = io.NopCloser(bytes.NewReader(bodyBuffer.Bytes()))
|
||||
r.Body = io.NopCloser(bytes.NewReader(body))
|
||||
|
||||
if bodyBuffer.Len() == 0 {
|
||||
if len(body) == 0 {
|
||||
return emptySHA256, nil
|
||||
}
|
||||
|
||||
return hex.EncodeToString(hasher.Sum(nil)), nil
|
||||
sum := sha256.Sum256(body)
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
// getContentSha256Cksum retrieves the "x-amz-content-sha256" header value.
|
||||
@@ -374,35 +372,29 @@ func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCh
|
||||
if trustedPeer {
|
||||
forwardedPrefix = r.Header.Get("X-Forwarded-Prefix")
|
||||
}
|
||||
// 9. Candidate paths: the escaped path as sent, then the decoded path that
|
||||
// an S3 client encodes once when signing; each with the forwarded prefix first.
|
||||
signaturePaths := []string{pathForSignature}
|
||||
if decodedPath, decodeErr := url.PathUnescape(pathForSignature); decodeErr == nil && decodedPath != pathForSignature {
|
||||
signaturePaths = append(signaturePaths, decodedPath)
|
||||
}
|
||||
if forwardedPrefix != "" {
|
||||
prefixedPaths := make([]string, 0, 2*len(signaturePaths))
|
||||
for _, p := range signaturePaths {
|
||||
prefixedPaths = append(prefixedPaths, buildPathWithForwardedPrefix(forwardedPrefix, p))
|
||||
}
|
||||
signaturePaths = append(prefixedPaths, signaturePaths...)
|
||||
}
|
||||
var matchedHost string
|
||||
hostCandidates:
|
||||
for i, hostCandidate := range extractHostHeaderCandidates(r, iam.externalHost) {
|
||||
if i > 0 && !replaceSignedHostHeader(extractedSignedHeaders, hostCandidate) {
|
||||
break
|
||||
}
|
||||
|
||||
// 9. Verify with the X-Forwarded-Prefix path first
|
||||
if forwardedPrefix != "" {
|
||||
cleanedPath := buildPathWithForwardedPrefix(forwardedPrefix, pathForSignature)
|
||||
calculatedSignature, errCode = verify(cleanedPath)
|
||||
if errCode == s3err.ErrNone {
|
||||
for _, p := range signaturePaths {
|
||||
if calculatedSignature, errCode = verify(p); errCode == s3err.ErrNone {
|
||||
matchedHost = hostCandidate
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// 10. Verify with the original path
|
||||
calculatedSignature, errCode = verify(pathForSignature)
|
||||
if errCode == s3err.ErrNone {
|
||||
matchedHost = hostCandidate
|
||||
break
|
||||
}
|
||||
|
||||
// 11. Retry with decoded path if signature used raw path encoding
|
||||
if decodedPath, decodeErr := url.PathUnescape(pathForSignature); decodeErr == nil && decodedPath != pathForSignature {
|
||||
calculatedSignature, errCode = verify(decodedPath)
|
||||
if errCode == s3err.ErrNone {
|
||||
matchedHost = hostCandidate
|
||||
break
|
||||
break hostCandidates
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -661,8 +653,12 @@ func extractV4AuthInfoFromHeader(r *http.Request) (*v4AuthInfo, s3err.ErrorCode)
|
||||
|
||||
hashedPayload := getContentSha256Cksum(r)
|
||||
if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil {
|
||||
sizeLimit := s3_constants.GetRequestBodyLimitFromContext(r, iamRequestBodyLimit)
|
||||
var hashErr error
|
||||
hashedPayload, hashErr = streamHashRequestBody(r, iamRequestBodyLimit)
|
||||
hashedPayload, hashErr = streamHashRequestBody(r, sizeLimit)
|
||||
if errors.Is(hashErr, errRequestBodyTooLarge) {
|
||||
return nil, s3err.ErrEntityTooLarge
|
||||
}
|
||||
if hashErr != nil {
|
||||
return nil, s3err.ErrInternalError
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -735,3 +736,35 @@ func TestVerifyV4SignatureUsesOriginalRequestPath(t *testing.T) {
|
||||
t.Fatalf("signed request rejected after path rewrite: %d", errCode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSignatureV4WithForwardedPrefixUnicodeKey guards against a signature
|
||||
// mismatch when an object key contains non-ASCII characters and the request is
|
||||
// received through a trusted reverse proxy that strips a path prefix. The
|
||||
// client signs the single percent-encoded path including the prefix, so the
|
||||
// server must decode the forwarded (already encoded) path before re-encoding it
|
||||
// together with the prefix.
|
||||
func TestSignatureV4WithForwardedPrefixUnicodeKey(t *testing.T) {
|
||||
iam := newTestIAMWithTrustedProxy()
|
||||
|
||||
bucket := "test-bucket"
|
||||
key := "たあいまd·.jpg"
|
||||
|
||||
// The proxy strips the X-Forwarded-Prefix and forwards the percent-encoded
|
||||
// object path (the server sees the encoded form, not the raw Unicode key).
|
||||
r, err := newTestRequest("GET", "https://example.com/"+bucket+"/"+url.PathEscape(key), 0, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test request: %v", err)
|
||||
}
|
||||
|
||||
r.Header.Set("X-Forwarded-Prefix", "/s3")
|
||||
r.Header.Set("Host", "example.com")
|
||||
r.RemoteAddr = "10.0.0.1:1234"
|
||||
|
||||
// The client signs the full URL path including the prefix, using the single
|
||||
// percent-encoded form of the Unicode key.
|
||||
signV4WithPath(r, "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "/s3/"+bucket+"/"+key)
|
||||
|
||||
if _, _, errCode := iam.doesSignatureMatch(r); errCode != s3err.ErrNone {
|
||||
t.Fatalf("signature verification failed for unicode key behind forwarded prefix: code %d", int(errCode))
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,8 @@ import (
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
|
||||
@@ -1820,18 +1822,10 @@ func TestIAMLargeBodySecurityLimit(t *testing.T) {
|
||||
", SignedHeaders=content-type;host;x-amz-date, Signature=dummysignature"
|
||||
req.Header.Set("Authorization", authHeader)
|
||||
|
||||
// The function should complete successfully but limit the body to 10 MiB
|
||||
identity, _, errCode := iam.doesSignatureMatch(req)
|
||||
|
||||
// Should get signature mismatch (dummy signature) but not internal error
|
||||
assert.Equal(t, s3err.ErrSignatureDoesNotMatch, errCode)
|
||||
assert.Equal(t, s3err.ErrEntityTooLarge, errCode)
|
||||
assert.Nil(t, identity)
|
||||
|
||||
// Verify the body was truncated to the limit (10 MiB)
|
||||
bodyBytes, err := io.ReadAll(req.Body)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 10*(1<<20), len(bodyBytes)) // Should be exactly 10 MiB
|
||||
assert.Equal(t, strings.Repeat("A", 10*(1<<20)), string(bodyBytes)) // All As, but truncated
|
||||
}
|
||||
|
||||
// Test the streaming hash implementation directly
|
||||
@@ -1922,30 +1916,65 @@ func TestStreamingVsNonStreamingConsistency(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test streaming with size limit enforcement
|
||||
func TestStreamingWithSizeLimit(t *testing.T) {
|
||||
// Create a payload larger than the limit
|
||||
largePayload := strings.Repeat("A", 11*(1<<20)) // 11 MiB
|
||||
|
||||
req, err := http.NewRequest("POST", "http://localhost:8111/", strings.NewReader(largePayload))
|
||||
exact := strings.Repeat("A", 1<<20)
|
||||
req, err := http.NewRequest("POST", "http://localhost:8111/", strings.NewReader(exact))
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Stream with the limit
|
||||
hash, err := streamHashRequestBody(req, iamRequestBodyLimit)
|
||||
hash, err := streamHashRequestBody(req, 1<<20)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, getSHA256Hash([]byte(exact)), hash)
|
||||
|
||||
// Verify the hash is computed for the truncated content (10 MiB)
|
||||
truncatedPayload := strings.Repeat("A", 10*(1<<20))
|
||||
expectedHash := sha256.Sum256([]byte(truncatedPayload))
|
||||
expectedHashStr := hex.EncodeToString(expectedHash[:])
|
||||
|
||||
assert.Equal(t, expectedHashStr, hash)
|
||||
|
||||
// Verify the body was truncated
|
||||
bodyBytes, err := io.ReadAll(req.Body)
|
||||
req, err = http.NewRequest("POST", "http://localhost:8111/", strings.NewReader(exact+"A"))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 10*(1<<20), len(bodyBytes))
|
||||
assert.Equal(t, truncatedPayload, string(bodyBytes))
|
||||
_, err = streamHashRequestBody(req, 1<<20)
|
||||
assert.ErrorIs(t, err, errRequestBodyTooLarge)
|
||||
}
|
||||
|
||||
func TestNonS3BodyHashLimitFollowsRoute(t *testing.T) {
|
||||
iam := &IdentityAccessManagement{}
|
||||
assert.NoError(t, iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{{
|
||||
Name: "vec",
|
||||
Credentials: []*iam_pb.Credential{{AccessKey: "vec_key", SecretKey: "vec_secret"}},
|
||||
Actions: []string{"Admin"},
|
||||
}},
|
||||
}))
|
||||
signer := v4.NewSigner(credentials.NewStaticCredentials("vec_key", "vec_secret", ""))
|
||||
|
||||
vectorsRoute := int64(s3_constants.S3VectorsMaxRequestBodyBytes)
|
||||
for _, tc := range []struct {
|
||||
service string
|
||||
routeLimit int64
|
||||
size int
|
||||
want s3err.ErrorCode
|
||||
}{
|
||||
{"s3vectors", vectorsRoute, 20 << 20, s3err.ErrNone},
|
||||
{"s3vectors", vectorsRoute, s3_constants.S3VectorsMaxRequestBodyBytes, s3err.ErrNone},
|
||||
{"s3vectors", vectorsRoute, s3_constants.S3VectorsMaxRequestBodyBytes + 1, s3err.ErrEntityTooLarge},
|
||||
// An s3vectors scope alone, on another route, keeps the 10 MiB limit.
|
||||
{"s3vectors", 0, iamRequestBodyLimit + 1, s3err.ErrEntityTooLarge},
|
||||
{"iam", 0, iamRequestBodyLimit + 1, s3err.ErrEntityTooLarge},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("%s_route%d_%d", tc.service, tc.routeLimit, tc.size), func(t *testing.T) {
|
||||
body := strings.Repeat("v", tc.size)
|
||||
req, err := http.NewRequest("POST", "http://localhost:8333/PutVectors", strings.NewReader(body))
|
||||
assert.NoError(t, err)
|
||||
if tc.routeLimit > 0 {
|
||||
req = req.WithContext(s3_constants.SetRequestBodyLimitInContext(req.Context(), tc.routeLimit))
|
||||
}
|
||||
_, err = signer.Sign(req, strings.NewReader(body), tc.service, "us-east-1", time.Now())
|
||||
assert.NoError(t, err)
|
||||
|
||||
identity, _, errCode := iam.doesSignatureMatch(req)
|
||||
assert.Equal(t, tc.want, errCode)
|
||||
if tc.want == s3err.ErrNone {
|
||||
assert.Equal(t, "vec", identity.Name)
|
||||
got, err := io.ReadAll(req.Body)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, len(body), len(got))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Benchmark streaming vs non-streaming memory usage
|
||||
|
||||
@@ -282,6 +282,10 @@ func (s *Server) authorizeCommit(ctx context.Context, bucketARN string, namespac
|
||||
Name: tableName,
|
||||
}, identityName)
|
||||
})
|
||||
if err == nil {
|
||||
bucketName, _ := s3tables.ParseBucketNameFromARN(bucketARN)
|
||||
err = s.tablesManager.CheckVectorNamespaceWrite(ctx, s.filerClient, bucketName, namespace)
|
||||
}
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -308,6 +308,12 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
|
||||
writeManagerError(w, authErr)
|
||||
return
|
||||
}
|
||||
// Registration confines the location to the table's namespace, but both
|
||||
// branches write, and stage-create vends credentials, before it runs.
|
||||
if err := s3tables.ValidateMetadataLocation(metadataLocation, bucketName, flattenNamespacePath(namespace)); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "BadRequestException", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Stage-create persists metadata in the internal staged area and skips S3Tables registration.
|
||||
if req.StageCreate {
|
||||
@@ -786,6 +792,9 @@ func (s *Server) mayUpdateTable(r *http.Request, bucketName string, namespace []
|
||||
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return s.tablesManager.AuthorizeUpdateTable(r.Context(), s3tables.NewManagerClient(client), updateReq, identityName)
|
||||
})
|
||||
if err == nil {
|
||||
err = s.tablesManager.CheckVectorNamespaceWrite(r.Context(), s.filerClient, bucketName, namespace)
|
||||
}
|
||||
if err != nil {
|
||||
glog.V(1).Infof("Iceberg: UpdateTable check for %s/%s failed, vending read-only: %v", bucketName, tableName, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
package iceberg
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
|
||||
)
|
||||
|
||||
type openAuthenticator struct{}
|
||||
|
||||
func (openAuthenticator) AuthenticateRequest(*http.Request) (string, interface{}, s3err.ErrorCode) {
|
||||
return s3tables.DefaultAccountID, nil, s3err.ErrNone
|
||||
}
|
||||
|
||||
func (openAuthenticator) DefaultAllow() bool { return true }
|
||||
|
||||
// An index in a vector bucket is a catalog table the Iceberg catalog can load,
|
||||
// but a caller holding every s3tables grant gets neither write credentials to
|
||||
// it nor a commit into it.
|
||||
func TestVectorIndexIsReadOnlyOverIceberg(t *testing.T) {
|
||||
filer := s3tablestest.Start(t)
|
||||
vendor := &stubVendor{credentials: &VendedCredentials{AccessKeyID: "ASIAEXAMPLE"}}
|
||||
s := NewServer(s3tables.NewManagerClient(filer.Client), openAuthenticator{})
|
||||
s.SetCredentialVendor(vendor)
|
||||
s.s3Endpoint = "http://127.0.0.1:8333"
|
||||
router := mux.NewRouter().SkipClean(true)
|
||||
s.RegisterRoutes(router)
|
||||
|
||||
admin := s3tables.NewManager()
|
||||
admin.SetTrusted(true)
|
||||
client := s3tables.NewManagerClient(filer.Client)
|
||||
arn, err := s3tables.BuildBucketARN(s3tables.DefaultRegion, s3tables.DefaultAccountID, "vec")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
metadata, _ := json.Marshal(map[string]any{"dimension": 4, "distanceMetric": "cosine"})
|
||||
for _, op := range []struct {
|
||||
name string
|
||||
req interface{}
|
||||
}{
|
||||
{"CreateTableBucket", &s3tables.CreateTableBucketRequest{Name: "vec", Format: s3tables.FormatLance}},
|
||||
{"CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}}},
|
||||
{"CreateTable", &s3tables.CreateTableRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace},
|
||||
Name: "idx", Format: s3tables.FormatLance, Metadata: &s3tables.TableMetadata{FullMetadata: metadata}}},
|
||||
} {
|
||||
if err := admin.Execute(t.Context(), client, op.name, op.req, nil, ""); err != nil {
|
||||
t.Fatalf("%s: %v", op.name, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/vec/namespaces/vector/tables/idx", nil)
|
||||
req.Header.Set(accessDelegationHeader, "vended-credentials")
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("load = %d (%s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
if vendor.gotPrefix != "vector/idx" || !vendor.gotReadOnly {
|
||||
t.Errorf("vended prefix %q read-only %v, want vector/idx read-only", vendor.gotPrefix, vendor.gotReadOnly)
|
||||
}
|
||||
|
||||
reqErr := s.authorizeCommit(t.Context(), arn, []string{s3tables.VectorIndexNamespace}, "idx", s3tables.DefaultAccountID)
|
||||
if reqErr == nil || reqErr.status != http.StatusForbidden {
|
||||
t.Errorf("commit authorization = %+v, want 403", reqErr)
|
||||
}
|
||||
}
|
||||
|
||||
// Stage-create registers nothing, so the catalog's own location and vector
|
||||
// namespace checks never see it. It must not hand out write credentials to an
|
||||
// index, whether the staged table names the index's location from another
|
||||
// namespace or sits in the vector namespace itself.
|
||||
func TestStageCreateCannotReachVectorIndex(t *testing.T) {
|
||||
filer := s3tablestest.Start(t)
|
||||
s := NewServer(s3tables.NewManagerClient(filer.Client), openAuthenticator{})
|
||||
s.s3Endpoint = "http://127.0.0.1:8333"
|
||||
router := mux.NewRouter().SkipClean(true)
|
||||
s.RegisterRoutes(router)
|
||||
|
||||
admin := s3tables.NewManager()
|
||||
admin.SetTrusted(true)
|
||||
client := s3tables.NewManagerClient(filer.Client)
|
||||
arn, err := s3tables.BuildBucketARN(s3tables.DefaultRegion, s3tables.DefaultAccountID, "vec")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, op := range []struct {
|
||||
name string
|
||||
req interface{}
|
||||
}{
|
||||
{"CreateTableBucket", &s3tables.CreateTableBucketRequest{Name: "vec", Format: s3tables.FormatLance}},
|
||||
{"CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}}},
|
||||
{"CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{"other"}}},
|
||||
{"CreateTable", &s3tables.CreateTableRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace},
|
||||
Name: "idx", Format: s3tables.FormatLance}},
|
||||
} {
|
||||
if err := admin.Execute(t.Context(), client, op.name, op.req, nil, ""); err != nil {
|
||||
t.Fatalf("%s: %v", op.name, err)
|
||||
}
|
||||
}
|
||||
|
||||
for name, tc := range map[string]struct {
|
||||
namespace, body string
|
||||
status int
|
||||
}{
|
||||
"index location from another namespace": {"other", `{"name":"x","stage-create":true,"location":"s3://vec/vector/idx"}`, http.StatusBadRequest},
|
||||
"vector namespace": {s3tables.VectorIndexNamespace, `{"name":"y","stage-create":true}`, http.StatusForbidden},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
vendor := &stubVendor{credentials: &VendedCredentials{AccessKeyID: "ASIAEXAMPLE"}}
|
||||
s.SetCredentialVendor(vendor)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/vec/namespaces/"+tc.namespace+"/tables", strings.NewReader(tc.body))
|
||||
req.Header.Set(accessDelegationHeader, "vended-credentials")
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != tc.status {
|
||||
t.Errorf("stage-create = %d (%s), want %d", rec.Code, rec.Body.String(), tc.status)
|
||||
}
|
||||
if vendor.gotPrefix != "" && !vendor.gotReadOnly {
|
||||
t.Errorf("vended write credentials to %q", vendor.gotPrefix)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package lance
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
jwt "github.com/golang-jwt/jwt/v5"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
)
|
||||
|
||||
func signAdminClaims(t *testing.T, key security.SigningKey, claims security.SeaweedFilerAdminClaims) string {
|
||||
t.Helper()
|
||||
token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(key))
|
||||
if err != nil {
|
||||
t.Fatalf("sign: %v", err)
|
||||
}
|
||||
return token
|
||||
}
|
||||
|
||||
func trustedSeenByHandler(t *testing.T, s *Server, token string) bool {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil)
|
||||
if token != "" {
|
||||
req.Header.Set(clusterTokenHeader, token)
|
||||
}
|
||||
trusted := false
|
||||
rec := httptest.NewRecorder()
|
||||
s.Auth(func(w http.ResponseWriter, r *http.Request) {
|
||||
trusted = isTrustedCaller(r.Context())
|
||||
w.WriteHeader(http.StatusOK)
|
||||
})(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d (%s), want 200", rec.Code, rec.Body.String())
|
||||
}
|
||||
return trusted
|
||||
}
|
||||
|
||||
func TestClusterTokenMarksTheCallerTrusted(t *testing.T) {
|
||||
key := security.SigningKey("cluster-key")
|
||||
s := NewServer(nil, openAuthenticator{})
|
||||
s.SetSigningKey(func() security.SigningKey { return key })
|
||||
|
||||
derived := clusterTokenKey(key)
|
||||
expired := security.SeaweedFilerAdminClaims{}
|
||||
expired.ExpiresAt = jwt.NewNumericDate(time.Now().Add(-time.Minute))
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
token string
|
||||
want bool
|
||||
}{
|
||||
{"no token", "", false},
|
||||
{"signed with the derived key", string(security.GenJwtForFilerAdmin(derived, 60)), true},
|
||||
{"signed with the cluster key itself", string(security.GenJwtForFilerAdmin(key, 60)), false},
|
||||
{"signed with another key", string(security.GenJwtForFilerAdmin(clusterTokenKey(security.SigningKey("other-key")), 60)), false},
|
||||
{"expired", signAdminClaims(t, derived, expired), false},
|
||||
{"session token", signAdminClaims(t, derived, security.SeaweedFilerAdminClaims{SessionId: "session"}), false},
|
||||
{"garbage", "not-a-token", false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := trustedSeenByHandler(t, s, tc.token); got != tc.want {
|
||||
t.Errorf("trusted = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
token := string(security.GenJwtForFilerAdmin(derived, 60))
|
||||
key = security.SigningKey("rotated-key")
|
||||
if trustedSeenByHandler(t, s, token) {
|
||||
t.Error("a token from before a key rotation is still trusted")
|
||||
}
|
||||
if !trustedSeenByHandler(t, s, string(security.GenJwtForFilerAdmin(clusterTokenKey(key), 60))) {
|
||||
t.Error("a token signed after a key rotation is not trusted")
|
||||
}
|
||||
|
||||
key = nil
|
||||
if trustedSeenByHandler(t, s, token) {
|
||||
t.Error("a server with no cluster key trusted a caller")
|
||||
}
|
||||
s.SetSigningKey(nil)
|
||||
if trustedSeenByHandler(t, s, token) {
|
||||
t.Error("a server with no key source trusted a caller")
|
||||
}
|
||||
}
|
||||
|
||||
// The Rust catalog client derives the same key; its tests pin this value too.
|
||||
func TestClusterTokenKeyDerivation(t *testing.T) {
|
||||
got := hex.EncodeToString(clusterTokenKey(security.SigningKey("cluster-key")))
|
||||
if want := "b38c0b66da41621f2a9b6de68e4239ddd0d2dd7b4ede01aa91bd34a17edc1021"; got != want {
|
||||
t.Errorf("clusterTokenKey = %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
@@ -49,8 +49,9 @@ func newTestServerWithOAuth() *Server {
|
||||
}
|
||||
|
||||
type mockS3Authenticator struct {
|
||||
called bool
|
||||
errCode s3err.ErrorCode
|
||||
called bool
|
||||
errCode s3err.ErrorCode
|
||||
authDisabled bool
|
||||
}
|
||||
|
||||
func (m *mockS3Authenticator) AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode) {
|
||||
@@ -63,6 +64,8 @@ func (m *mockS3Authenticator) AuthenticateRequest(r *http.Request) (string, inte
|
||||
|
||||
func (m *mockS3Authenticator) DefaultAllow() bool { return false }
|
||||
|
||||
func (m *mockS3Authenticator) AuthEnabled() bool { return !m.authDisabled }
|
||||
|
||||
func mintTestToken(t *testing.T, accessKey, secret string, issuedAt, expiresAt time.Time) string {
|
||||
t.Helper()
|
||||
key := deriveSigningKey(accessKey, secret)
|
||||
@@ -316,6 +319,32 @@ func TestAuthApiKeyInvalid(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Without identities the S3 port takes any signature, so the catalog takes any
|
||||
// key or token and leaves the caller to the S3 authenticator.
|
||||
func TestAuthTakesAnyKeyWhenS3AuthIsOff(t *testing.T) {
|
||||
s := newTestServerWithOAuth()
|
||||
s.authenticator = &mockS3Authenticator{authDisabled: true}
|
||||
|
||||
for header, value := range map[string]string{
|
||||
"x-api-key": "unknown:key",
|
||||
"Authorization": "Bearer not-a-token",
|
||||
} {
|
||||
var gotIdentity string
|
||||
handler := s.Auth(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotIdentity = s3_constants.GetIdentityNameFromContext(r)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
})
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil)
|
||||
req.Header.Set(header, value)
|
||||
rec := httptest.NewRecorder()
|
||||
handler(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK || gotIdentity != "s3user" {
|
||||
t.Fatalf("%s: status = %d, identity = %q, want 200 as s3user", header, rec.Code, gotIdentity)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthNoBearerStillUsesS3Authenticator(t *testing.T) {
|
||||
s := newTestServerWithOAuth()
|
||||
auth := &mockS3Authenticator{errCode: s3err.ErrNone}
|
||||
|
||||
@@ -135,24 +135,35 @@ func (s *Server) handleDeclareTable(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.createTable(r, bucket, ns, name, location); err != nil {
|
||||
_, lookupErr := s.loadLanceTable(r, bucket, ns, name)
|
||||
location, err = s.createTable(r, bucket, ns, name, location)
|
||||
if err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
if err := s.writeMarker(r, location, reservedMarker); err != nil {
|
||||
// The catalog entry is the authority; the marker only mirrors it for
|
||||
// clients that read the storage prefix directly.
|
||||
glog.V(1).Infof("lance: could not write %s for %s: %v", reservedMarker, location, err)
|
||||
// A declare that creates the table may write it: a Lance commit is a direct
|
||||
// S3 write that never revisits the catalog, so the create authorization is
|
||||
// the write evidence. One that only finds the table answers like describe.
|
||||
var writable bool
|
||||
if isNotFound(lookupErr) {
|
||||
writable = s.mayWriteNamespace(r, bucket, ns)
|
||||
} else {
|
||||
writable = s.mayUpdateTable(r, bucket, ns, name)
|
||||
}
|
||||
// Declaring a name that was deregistered brings it back, the same way
|
||||
// registering it does.
|
||||
if err := s.removeMarker(r, location, deregisteredMarker); err != nil {
|
||||
glog.V(2).Infof("lance: could not clear %s for %s: %v", deregisteredMarker, location, err)
|
||||
if writable {
|
||||
if err := s.writeMarker(r, location, reservedMarker); err != nil {
|
||||
// The catalog entry is the authority; the marker only mirrors it for
|
||||
// clients that read the storage prefix directly.
|
||||
glog.V(1).Infof("lance: could not write %s for %s: %v", reservedMarker, location, err)
|
||||
}
|
||||
// Declaring a name that was deregistered brings it back, the same way
|
||||
// registering it does.
|
||||
if err := s.removeMarker(r, location, deregisteredMarker); err != nil {
|
||||
glog.V(2).Infof("lance: could not clear %s for %s: %v", deregisteredMarker, location, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A Lance commit is a direct S3 write that never revisits the catalog, so
|
||||
// the create authorization above is the write evidence for the dataset.
|
||||
options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials), func() bool { return true })
|
||||
options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials), func() bool { return writable })
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
|
||||
return
|
||||
@@ -353,7 +364,7 @@ func (s *Server) handleRegisterTable(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
default:
|
||||
if err := s.createTable(r, bucket, ns, name, location); err != nil {
|
||||
if location, err = s.createTable(r, bucket, ns, name, location); err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
@@ -545,7 +556,9 @@ func (s *Server) loadLanceTable(r *http.Request, bucket string, ns []string, nam
|
||||
return &lanceTable{location: location, versionToken: resp.VersionToken}, nil
|
||||
}
|
||||
|
||||
func (s *Server) createTable(r *http.Request, bucket string, ns []string, name, location string) error {
|
||||
// createTable returns the location the catalog holds for the table. When the
|
||||
// name already exists that is its own location, not the one asked for.
|
||||
func (s *Server) createTable(r *http.Request, bucket string, ns []string, name, location string) (string, error) {
|
||||
req := &s3tables.CreateTableRequest{
|
||||
TableBucketARN: bucketARN(bucket),
|
||||
Namespace: ns,
|
||||
@@ -556,7 +569,13 @@ func (s *Server) createTable(r *http.Request, bucket string, ns []string, name,
|
||||
MetadataLocation: location,
|
||||
}
|
||||
var resp s3tables.CreateTableResponse
|
||||
return s.execute(r, "CreateTable", req, &resp)
|
||||
if err := s.execute(r, "CreateTable", req, &resp); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if resp.MetadataLocation == "" {
|
||||
return tableLocation(bucket, ns, name), nil
|
||||
}
|
||||
return resp.MetadataLocation, nil
|
||||
}
|
||||
|
||||
// repointTable moves an existing entry to another dataset location without
|
||||
|
||||
@@ -24,6 +24,14 @@ func (openAuthenticator) AuthenticateRequest(*http.Request) (string, interface{}
|
||||
|
||||
func (openAuthenticator) DefaultAllow() bool { return true }
|
||||
|
||||
func (openAuthenticator) AuthEnabled() bool { return false }
|
||||
|
||||
// signedAuthenticator admits the same caller on a gateway that checks
|
||||
// credentials.
|
||||
type signedAuthenticator struct{ openAuthenticator }
|
||||
|
||||
func (signedAuthenticator) AuthEnabled() bool { return true }
|
||||
|
||||
type testHarness struct {
|
||||
router *mux.Router
|
||||
filer *s3tablestest.MemFiler
|
||||
@@ -134,6 +142,28 @@ func decode[T any](t *testing.T, recorder *httptest.ResponseRecorder) T {
|
||||
return out
|
||||
}
|
||||
|
||||
// While the S3 port takes unsigned requests the catalog says so, and a client
|
||||
// with no credentials of its own still reaches the data. Once auth is on it
|
||||
// has to sign again.
|
||||
func TestStorageOptionsSkipSignatureOnlyWhileS3AuthIsOff(t *testing.T) {
|
||||
h := newTestHarness(t)
|
||||
h.createBucket(t, "open", s3tables.FormatLance)
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/open$ns/create", `{}`, http.StatusOK)
|
||||
|
||||
declared := decode[DeclareTableResponse](t,
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/open$ns$t/declare", `{}`, http.StatusOK))
|
||||
if declared.StorageOptions["aws_skip_signature"] != "true" {
|
||||
t.Fatalf("auth off: storage options = %v, want aws_skip_signature", declared.StorageOptions)
|
||||
}
|
||||
|
||||
h.server.authenticator = signedAuthenticator{}
|
||||
described := decode[DescribeTableResponse](t,
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/open$ns$t/describe", `{}`, http.StatusOK))
|
||||
if _, ok := described.StorageOptions["aws_skip_signature"]; ok {
|
||||
t.Fatalf("auth on: storage options = %v, want no aws_skip_signature", described.StorageOptions)
|
||||
}
|
||||
}
|
||||
|
||||
// The lifecycle a Lance client drives: create the namespace, declare the table
|
||||
// before any data exists, resolve it to a location, then deregister and bring it
|
||||
// back by registering the same location.
|
||||
@@ -415,3 +445,48 @@ func TestDataPlaneIsUnsupported(t *testing.T) {
|
||||
t.Fatalf("error code = %d, want %d", got.Code, codeUnsupported)
|
||||
}
|
||||
}
|
||||
|
||||
// createOnlyAuthenticator signs in an identity that may create and read tables
|
||||
// but not update them.
|
||||
type createOnlyAuthenticator struct{}
|
||||
|
||||
func (createOnlyAuthenticator) AuthenticateRequest(*http.Request) (string, interface{}, s3err.ErrorCode) {
|
||||
return "creator", &struct {
|
||||
Name string
|
||||
Actions []string
|
||||
}{Name: "creator", Actions: []string{"s3tables:CreateTable", "s3tables:GetTable"}}, s3err.ErrNone
|
||||
}
|
||||
|
||||
func (createOnlyAuthenticator) DefaultAllow() bool { return false }
|
||||
|
||||
func (createOnlyAuthenticator) AuthEnabled() bool { return true }
|
||||
|
||||
// The right to create a name is the write evidence for a table the declare
|
||||
// creates, but not for one it only finds.
|
||||
func TestDeclareVendsWriteOnlyForATableItCreates(t *testing.T) {
|
||||
h := newTestHarness(t)
|
||||
h.createBucket(t, "lnc", s3tables.FormatLance)
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/lnc$ns/create", `{}`, http.StatusOK)
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/lnc$ns$theirs/declare", `{}`, http.StatusOK)
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/lnc$ns$theirs/deregister", `{}`, http.StatusOK)
|
||||
|
||||
server := NewServer(s3tables.NewManagerClient(h.filer.Client), createOnlyAuthenticator{})
|
||||
server.SetCredentialVendor(recordingVendor{})
|
||||
router := mux.NewRouter().SkipClean(true)
|
||||
server.RegisterRoutes(router)
|
||||
creator := &testHarness{router: router, filer: h.filer, admin: h.admin, server: server}
|
||||
|
||||
for _, tc := range []struct{ name, want string }{
|
||||
{"theirs", "read-only"},
|
||||
{"mine", "read-write"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := creator.vendedKey(t, "/v1/table/lnc$ns$"+tc.name+"/declare", ""); got != tc.want {
|
||||
t.Errorf("vended %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
if h.filer.Get("/buckets/lnc/ns/theirs", deregisteredMarker) == nil {
|
||||
t.Error("a declare that may not write the table brought it back")
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,8 @@ package lance
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -12,6 +14,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
)
|
||||
|
||||
// FilerClient provides access to the filer for storage operations.
|
||||
@@ -22,6 +25,8 @@ type FilerClient interface {
|
||||
type S3Authenticator interface {
|
||||
AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode)
|
||||
DefaultAllow() bool
|
||||
// AuthEnabled reports whether the S3 server checks credentials at all.
|
||||
AuthEnabled() bool
|
||||
}
|
||||
|
||||
// CredentialValidator validates S3 access key / secret key pairs
|
||||
@@ -61,6 +66,7 @@ type Server struct {
|
||||
credentialVendor CredentialVendor
|
||||
s3Endpoint string
|
||||
s3Region string
|
||||
signingKey func() security.SigningKey
|
||||
}
|
||||
|
||||
// NewServer creates a Lance namespace server over the given filer.
|
||||
@@ -89,6 +95,13 @@ func (s *Server) SetCredentialVendor(vendor CredentialVendor) {
|
||||
s.credentialVendor = vendor
|
||||
}
|
||||
|
||||
// SetSigningKey sets where to read the cluster key (jwt.filer_signing.key)
|
||||
// that a trusted caller's clusterTokenHeader is checked against. It is read on
|
||||
// every check, so a reloaded key takes effect at once.
|
||||
func (s *Server) SetSigningKey(signingKey func() security.SigningKey) {
|
||||
s.signingKey = signingKey
|
||||
}
|
||||
|
||||
// SetS3Endpoint configures the S3 endpoint advertised in storage_options so a
|
||||
// client can reach the dataset without separately discovering the S3 address.
|
||||
func (s *Server) SetS3Endpoint(endpoint string) {
|
||||
@@ -166,17 +179,27 @@ func loggingMiddleware(next http.Handler) http.Handler {
|
||||
})
|
||||
}
|
||||
|
||||
// authEnabled is false when the S3 server has no identities and takes any
|
||||
// signature. The catalog then takes any key too, and leaves the caller to the
|
||||
// S3 authenticator.
|
||||
func (s *Server) authEnabled() bool {
|
||||
return s.authenticator == nil || s.authenticator.AuthEnabled()
|
||||
}
|
||||
|
||||
// Auth authenticates the caller and puts the identity in the request context.
|
||||
// Lance clients authenticate the catalog with a Bearer token or an x-api-key
|
||||
// header; the S3 authenticator stays for callers that can SigV4-sign.
|
||||
func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.hasClusterToken(r) {
|
||||
r = r.WithContext(withTrustedCaller(r.Context()))
|
||||
}
|
||||
// A request carrying a Bearer token is a Lance REST client. An invalid
|
||||
// or expired token gets 401 immediately; falling through to the S3
|
||||
// authenticator would parse the header as SigV4 and fail with a
|
||||
// different error that clients cannot act on.
|
||||
// The auth scheme is case-insensitive (RFC 7235).
|
||||
if strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") {
|
||||
if s.authEnabled() && strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") {
|
||||
if identityName, identity, accessKey, secretKey, ok := s.authenticateBearerKey(r); ok {
|
||||
ctx := r.Context()
|
||||
ctx = s3_constants.SetIdentityNameInContext(ctx, identityName)
|
||||
@@ -194,7 +217,7 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
if apiKey := r.Header.Get("x-api-key"); apiKey != "" {
|
||||
if apiKey := r.Header.Get("x-api-key"); apiKey != "" && s.authEnabled() {
|
||||
if identityName, identity, ok := s.authenticateApiKey(apiKey); ok {
|
||||
accessKey, secretKey, _ := strings.Cut(apiKey, ":")
|
||||
ctx := r.Context()
|
||||
@@ -250,3 +273,46 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
|
||||
handler(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
// clusterTokenHeader carries a token signed with clusterTokenKey. The vector
|
||||
// engine and the Lance maintenance worker send it next to their identity's
|
||||
// credentials, and only they may write index data in a vector bucket.
|
||||
const clusterTokenHeader = "X-Seaweedfs-Cluster-Token"
|
||||
|
||||
// clusterTokenKey derives the key cluster tokens are signed with from the
|
||||
// cluster key, so a token captured on its way here does not authorize the
|
||||
// admin calls that take the cluster key itself.
|
||||
func clusterTokenKey(key security.SigningKey) security.SigningKey {
|
||||
mac := hmac.New(sha256.New, key)
|
||||
mac.Write([]byte("seaweedfs-lance-cluster-token"))
|
||||
return mac.Sum(nil)
|
||||
}
|
||||
|
||||
type trustedCallerKey struct{}
|
||||
|
||||
func withTrustedCaller(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, trustedCallerKey{}, true)
|
||||
}
|
||||
|
||||
func isTrustedCaller(ctx context.Context) bool {
|
||||
trusted, _ := ctx.Value(trustedCallerKey{}).(bool)
|
||||
return trusted
|
||||
}
|
||||
|
||||
// hasClusterToken reports whether the request proves it holds the cluster key.
|
||||
// Without a configured key no caller can.
|
||||
func (s *Server) hasClusterToken(r *http.Request) bool {
|
||||
token := r.Header.Get(clusterTokenHeader)
|
||||
if token == "" || s.signingKey == nil {
|
||||
return false
|
||||
}
|
||||
key := s.signingKey()
|
||||
if len(key) == 0 {
|
||||
return false
|
||||
}
|
||||
if security.IsFilerAdminToken(clusterTokenKey(key), security.EncodedJwt(token)) {
|
||||
return true
|
||||
}
|
||||
glog.V(1).Infof("lance: %s from %s is not signed with a key derived from jwt.filer_signing.key", clusterTokenHeader, r.RemoteAddr)
|
||||
return false
|
||||
}
|
||||
@@ -149,6 +149,11 @@ func (s *Server) storageOptions(r *http.Request, bucket, location string, vend b
|
||||
if s.s3Region != "" {
|
||||
options["aws_region"] = s.s3Region
|
||||
}
|
||||
if !s.authEnabled() {
|
||||
// The S3 port takes unsigned requests, and a client with no
|
||||
// credentials of its own would otherwise fail looking for some.
|
||||
options["aws_skip_signature"] = "true"
|
||||
}
|
||||
|
||||
if !vend || s.credentialVendor == nil {
|
||||
return options, nil
|
||||
@@ -177,6 +182,9 @@ func (s *Server) storageOptions(r *http.Request, bucket, location string, vend b
|
||||
// credentials would cover, the same gate the Iceberg catalog applies so a
|
||||
// read-only caller never gets write-capable storage_options.
|
||||
func (s *Server) mayUpdateTable(r *http.Request, bucket string, ns []string, name string) bool {
|
||||
if !s.mayWriteNamespace(r, bucket, ns) {
|
||||
return false
|
||||
}
|
||||
req := &s3tables.UpdateTableRequest{
|
||||
TableBucketARN: bucketARN(bucket),
|
||||
Namespace: ns,
|
||||
@@ -192,6 +200,20 @@ func (s *Server) mayUpdateTable(r *http.Request, bucket string, ns []string, nam
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// mayWriteNamespace keeps the vector namespace of a vector bucket out of reach
|
||||
// of s3tables grants: its index data is written only by trusted callers, and
|
||||
// the vector bucket policy gates everyone else through S3 Vectors.
|
||||
func (s *Server) mayWriteNamespace(r *http.Request, bucket string, ns []string) bool {
|
||||
if isTrustedCaller(r.Context()) {
|
||||
return true
|
||||
}
|
||||
err := s.tablesManager.CheckVectorNamespaceWrite(r.Context(), s.filerClient, bucket, ns)
|
||||
if err != nil {
|
||||
glog.V(1).Infof("lance: %s/%s is read-only to this caller: %v", bucket, strings.Join(ns, "."), err)
|
||||
}
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// locationPrefix strips the s3://bucket/ part of a location, leaving the key
|
||||
// prefix a credential is scoped to.
|
||||
func locationPrefix(location string) string {
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
package lance
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
)
|
||||
|
||||
// seedVectorIndex creates bucket/vector/idx through the trusted path, the way
|
||||
@@ -75,3 +79,87 @@ func TestVectorNamespaceOfUndeclaredBucketIsWritableOverLance(t *testing.T) {
|
||||
h.mustDo(t, http.MethodPost, idx+"/register", `{"location":"s3://old/vector/idx"}`, http.StatusOK)
|
||||
h.mustDo(t, http.MethodPost, idx+"/drop", `{}`, http.StatusOK)
|
||||
}
|
||||
|
||||
// recordingVendor vends a credential whose key says whether it may write and
|
||||
// whose secret is the prefix it covers.
|
||||
type recordingVendor struct{}
|
||||
|
||||
func (recordingVendor) VendTableCredentials(_ context.Context, _, _, prefix string, readOnly bool) (*VendedCredentials, error) {
|
||||
if readOnly {
|
||||
return &VendedCredentials{AccessKeyID: "read-only", SecretAccessKey: prefix}, nil
|
||||
}
|
||||
return &VendedCredentials{AccessKeyID: "read-write", SecretAccessKey: prefix}, nil
|
||||
}
|
||||
|
||||
func (h *testHarness) vendedKey(t *testing.T, target, token string) string {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodPost, target, strings.NewReader(`{"vend_credentials":true}`))
|
||||
if token != "" {
|
||||
req.Header.Set(clusterTokenHeader, token)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
h.router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("POST %s = %d (%s)", target, rec.Code, rec.Body.String())
|
||||
}
|
||||
return decode[DescribeTableResponse](t, rec).StorageOptions["aws_access_key_id"]
|
||||
}
|
||||
|
||||
// A caller holding every s3tables grant still gets read-only credentials to a
|
||||
// vector bucket's index data; only the cluster's own components can write it.
|
||||
func TestVectorNamespaceVendsWriteCredentialsOnlyToTrustedCallers(t *testing.T) {
|
||||
h := newTestHarness(t)
|
||||
key := security.SigningKey("cluster-key")
|
||||
h.server.SetSigningKey(func() security.SigningKey { return key })
|
||||
h.server.SetCredentialVendor(recordingVendor{})
|
||||
h.seedVectorIndex(t, "vec", s3tables.FormatLance)
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/vec$other$tbl/declare", `{}`, http.StatusOK)
|
||||
h.seedVectorIndex(t, "old", "")
|
||||
|
||||
h.vendedKey(t, "/v1/table/vec$vector$idx/declare", "")
|
||||
if h.filer.Get("/buckets/vec/vector/idx", reservedMarker) != nil {
|
||||
t.Error("an untrusted declare wrote into the index")
|
||||
}
|
||||
|
||||
trusted := string(security.GenJwtForFilerAdmin(clusterTokenKey(key), 60))
|
||||
forged := string(security.GenJwtForFilerAdmin(key, 60))
|
||||
for _, tc := range []struct {
|
||||
name, target, token, want string
|
||||
}{
|
||||
{"describe the index", "/v1/table/vec$vector$idx/describe", "", "read-only"},
|
||||
{"declare the index", "/v1/table/vec$vector$idx/declare", "", "read-only"},
|
||||
{"describe with an admin token", "/v1/table/vec$vector$idx/describe", forged, "read-only"},
|
||||
{"trusted describe", "/v1/table/vec$vector$idx/describe", trusted, "read-write"},
|
||||
{"trusted declare", "/v1/table/vec$vector$idx/declare", trusted, "read-write"},
|
||||
{"another namespace", "/v1/table/vec$other$tbl/describe", "", "read-write"},
|
||||
{"bucket from before formats", "/v1/table/old$vector$idx/describe", "", "read-write"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := h.vendedKey(t, tc.target, tc.token); got != tc.want {
|
||||
t.Errorf("vended %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Declaring a name that already exists answers for that table, so naming the
|
||||
// index's location on another table gets no credentials to the index.
|
||||
func TestDeclareOfAnExistingTableKeepsItsLocation(t *testing.T) {
|
||||
h := newTestHarness(t)
|
||||
h.server.SetCredentialVendor(recordingVendor{})
|
||||
h.seedVectorIndex(t, "vec", s3tables.FormatLance)
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/vec$other$tbl/declare", `{}`, http.StatusOK)
|
||||
|
||||
rec := h.mustDo(t, http.MethodPost, "/v1/table/vec$other$tbl/declare",
|
||||
`{"location":"s3://vec/vector/idx","vend_credentials":true}`, http.StatusOK)
|
||||
resp := decode[DeclareTableResponse](t, rec)
|
||||
if resp.Location != "s3://vec/other/tbl" {
|
||||
t.Errorf("location = %q, want the table's own s3://vec/other/tbl", resp.Location)
|
||||
}
|
||||
if got := resp.StorageOptions["aws_secret_access_key"]; got != "other/tbl" {
|
||||
t.Errorf("credentials cover %q, want other/tbl", got)
|
||||
}
|
||||
if h.filer.Get("/buckets/vec/vector/idx", reservedMarker) != nil {
|
||||
t.Error("the declare wrote into the index")
|
||||
}
|
||||
}
|
||||
@@ -213,12 +213,28 @@ func (s3a *S3ApiServer) getEncryptionConfiguration(bucket string) (*s3_pb.Encryp
|
||||
}
|
||||
|
||||
if metadata.Encryption == nil {
|
||||
return nil, s3err.ErrNoSuchBucketEncryptionConfiguration
|
||||
return s3a.declaredEncryption(bucket)
|
||||
}
|
||||
|
||||
return metadata.Encryption, s3err.ErrNone
|
||||
}
|
||||
|
||||
// declaredEncryption is the encryption a table bucket's record declares. It
|
||||
// applies while the bucket has no S3 default encryption of its own.
|
||||
func (s3a *S3ApiServer) declaredEncryption(bucket string) (*s3_pb.EncryptionConfiguration, s3err.ErrorCode) {
|
||||
if s3a.bucketConfigCache == nil {
|
||||
return nil, s3err.ErrNoSuchBucketEncryptionConfiguration
|
||||
}
|
||||
config, errCode := s3a.getBucketConfig(bucket)
|
||||
if errCode != s3err.ErrNone {
|
||||
return nil, errCode
|
||||
}
|
||||
if config.DeclaredEncryption == nil {
|
||||
return nil, s3err.ErrNoSuchBucketEncryptionConfiguration
|
||||
}
|
||||
return config.DeclaredEncryption, s3err.ErrNone
|
||||
}
|
||||
|
||||
// updateEncryptionConfiguration updates the encryption configuration for a bucket
|
||||
func (s3a *S3ApiServer) updateEncryptionConfiguration(bucket string, encryptionConfig *s3_pb.EncryptionConfiguration) s3err.ErrorCode {
|
||||
// Update using structured API
|
||||
@@ -249,6 +265,10 @@ func (s3a *S3ApiServer) removeEncryptionConfiguration(bucket string) s3err.Error
|
||||
}
|
||||
|
||||
if metadata.Encryption == nil {
|
||||
// A bucket's declared encryption is fixed when the bucket is created.
|
||||
if declared, _ := s3a.declaredEncryption(bucket); declared != nil {
|
||||
return s3err.ErrMethodNotAllowed
|
||||
}
|
||||
return s3err.ErrNoSuchBucketEncryptionConfiguration
|
||||
}
|
||||
|
||||
|
||||
@@ -359,6 +359,7 @@ const (
|
||||
contextKeyOriginalRequestPath contextKey = "s3-original-request-path"
|
||||
contextKeyAccessKeyId contextKey = "s3-access-key-id"
|
||||
contextKeyStoredAccessKeyId contextKey = "s3-stored-access-key-id"
|
||||
contextKeyRequestBodyLimit contextKey = "s3-request-body-limit"
|
||||
)
|
||||
|
||||
// identityHolder is a mutable container for the authenticated identity name,
|
||||
@@ -529,6 +530,20 @@ func GetOriginalRequestPathFromContext(r *http.Request) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// SetRequestBodyLimitInContext sets how much of the body SigV4 verification
|
||||
// reads. The route sets it: the credential scope is not verified yet.
|
||||
func SetRequestBodyLimitInContext(ctx context.Context, limit int64) context.Context {
|
||||
return context.WithValue(ctx, contextKeyRequestBodyLimit, limit)
|
||||
}
|
||||
|
||||
// GetRequestBodyLimitFromContext returns the route's limit, or def.
|
||||
func GetRequestBodyLimitFromContext(r *http.Request, def int64) int64 {
|
||||
if limit, ok := r.Context().Value(contextKeyRequestBodyLimit).(int64); ok {
|
||||
return limit
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// SetAccessKeyIdInContext stores the access key id the request authenticated
|
||||
// with: a static key's id, or an STS session's temporary key id. Several keys
|
||||
// can belong to one identity, so the identity name alone cannot tell their
|
||||
|
||||
@@ -13,6 +13,9 @@ var (
|
||||
Separator = ":"
|
||||
)
|
||||
|
||||
// S3VectorsMaxRequestBodyBytes caps the body of an S3 Vectors API request.
|
||||
const S3VectorsMaxRequestBodyBytes = 32 << 20
|
||||
|
||||
func Concat(elements ...string) string {
|
||||
return strings.Join(elements, Separator)
|
||||
}
|
||||
@@ -778,6 +778,13 @@ func (km *SSES3KeyManager) StoreKey(key *SSES3Key) {
|
||||
// The DEK is encrypted with the super key and stored in object metadata
|
||||
}
|
||||
|
||||
// IsConfigured reports whether a KEK is loaded, i.e. SSE-S3 can encrypt.
|
||||
func (km *SSES3KeyManager) IsConfigured() bool {
|
||||
km.mu.RLock()
|
||||
defer km.mu.RUnlock()
|
||||
return len(km.superKey) > 0
|
||||
}
|
||||
|
||||
// GetMasterKey returns a derived key from the master KEK for STS signing
|
||||
// This uses HKDF to isolate the STS security domain from the SSE-S3 domain
|
||||
func (km *SSES3KeyManager) GetMasterKey() []byte {
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
// BucketConfig represents cached bucket configuration. Only fields parsed
|
||||
@@ -56,6 +57,7 @@ type BucketConfig struct {
|
||||
LifecycleTransitionMinSize string
|
||||
Tags map[string]string // parsed from entry content
|
||||
Encryption *s3_pb.EncryptionConfiguration // parsed from entry content
|
||||
DeclaredEncryption *s3_pb.EncryptionConfiguration // declared by a table bucket's record
|
||||
KMSKeyCache *BucketKMSCache // Per-bucket KMS key cache for SSE-KMS operations
|
||||
LastModified time.Time
|
||||
}
|
||||
@@ -441,6 +443,9 @@ func (s3a *S3ApiServer) newBucketConfigFromEntry(bucket string, entry *filer_pb.
|
||||
config.ObjectLockConfig = objectLockConfig
|
||||
}
|
||||
config.BucketPolicy = loadBucketPolicyFromExtended(entry, bucket)
|
||||
if algorithm := s3tables.DeclaredBucketEncryption(entry); algorithm != "" {
|
||||
config.DeclaredEncryption = &s3_pb.EncryptionConfiguration{SseAlgorithm: algorithm}
|
||||
}
|
||||
|
||||
if lifecycleXML, exists := entry.Extended[bucketLifecycleConfigurationXMLKey]; exists && len(lifecycleXML) > 0 {
|
||||
config.LifecycleXML = lifecycleXML
|
||||
|
||||
@@ -2,6 +2,7 @@ package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
// Issue 11647: Bucket default encryption (SSE-S3) must be applied when
|
||||
@@ -218,3 +220,72 @@ func TestPutObjectUnderUploadsFolderKeepsBucketDefaultEncryption(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A vector bucket is a LANCE table bucket; the AES256 its record declares is
|
||||
// the bucket's default encryption. An ICEBERG bucket's declaration is not
|
||||
// applied, because its catalog reads client manifests straight from the filer.
|
||||
func TestPutObjectAppliesTableBucketDeclaredEncryption(t *testing.T) {
|
||||
km := GetSSES3KeyManager()
|
||||
oldSuperKey := km.superKey
|
||||
km.superKey = make([]byte, 32)
|
||||
for i := range km.superKey {
|
||||
km.superKey[i] = byte(i + 1)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
km.superKey = oldSuperKey
|
||||
})
|
||||
|
||||
volume := startFakeVolumeServer(t)
|
||||
filerImpl := &ambiguousPutFiler{
|
||||
volume: volume,
|
||||
entries: map[string]*filer_pb.Entry{},
|
||||
apply: true,
|
||||
}
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, filerImpl))
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
for bucket, format := range map[string]string{"lance": s3tables.FormatLance, "ice": s3tables.FormatIceberg} {
|
||||
s3a.bucketConfigCache.Set(bucket, s3a.newBucketConfigFromEntry(bucket, &filer_pb.Entry{
|
||||
Name: bucket,
|
||||
Extended: map[string][]byte{
|
||||
s3tables.ExtendedKeyTableBucket: []byte("true"),
|
||||
s3tables.ExtendedKeyMetadata: []byte(`{"name":"` + bucket + `","format":"` + format +
|
||||
`","encryptionConfiguration":{"sseAlgorithm":"AES256"}}`),
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
cfg, err := s3a.GetBucketEncryptionConfig("lance")
|
||||
if err != nil || cfg.SseAlgorithm != EncryptionTypeAES256 {
|
||||
t.Fatalf("GetBucketEncryptionConfig(lance) = %v, %v; want AES256", cfg, err)
|
||||
}
|
||||
if metadata, err := s3a.GetBucketMetadata("lance"); err != nil || metadata.Encryption != nil {
|
||||
t.Fatalf("declared encryption leaked into the stored bucket metadata: %v, %v", metadata, err)
|
||||
}
|
||||
if _, err := s3a.GetBucketEncryptionConfig("ice"); !errors.Is(err, ErrNoEncryptionConfig) {
|
||||
t.Fatalf("GetBucketEncryptionConfig(ice) error = %v, want ErrNoEncryptionConfig", err)
|
||||
}
|
||||
if code := s3a.removeEncryptionConfiguration("lance"); code != s3err.ErrMethodNotAllowed {
|
||||
t.Fatalf("DeleteBucketEncryption(lance) = %v, want MethodNotAllowed for a declared encryption", code)
|
||||
}
|
||||
if code := s3a.removeEncryptionConfiguration("ice"); code != s3err.ErrNoSuchBucketEncryptionConfiguration {
|
||||
t.Fatalf("DeleteBucketEncryption(ice) = %v, want NoSuchBucketEncryptionConfiguration", code)
|
||||
}
|
||||
|
||||
for bucket, want := range map[string]filer_pb.SSEType{"lance": filer_pb.SSEType_SSE_S3, "ice": filer_pb.SSEType_NONE} {
|
||||
filePath := "/buckets/" + bucket + "/vector/idx/data/0.lance"
|
||||
r := httptest.NewRequest(http.MethodPut, "/"+bucket+"/vector/idx/data/0.lance", nil)
|
||||
_, code, _ := s3a.putToFiler(r, filePath, strings.NewReader("lance data file"), bucket, "vector/idx/data/0.lance", 1, 0, nil, false, "")
|
||||
if code != s3err.ErrNone {
|
||||
t.Fatalf("putToFiler(%s) = %v, want %v", bucket, code, s3err.ErrNone)
|
||||
}
|
||||
entry := filerImpl.entries[filePath]
|
||||
if entry == nil || len(entry.Chunks) == 0 {
|
||||
t.Fatalf("no entry with chunks at %s", filePath)
|
||||
}
|
||||
for i, chunk := range entry.Chunks {
|
||||
if chunk.SseType != want {
|
||||
t.Errorf("%s chunk %d: SseType %v, want %v", bucket, i, chunk.SseType, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -342,6 +342,8 @@ func iamAuthErrorCode(errCode s3err.ErrorCode) string {
|
||||
return "RequestTimeTooSkewed"
|
||||
case s3err.ErrInternalError:
|
||||
return iam.ErrCodeServiceFailureException
|
||||
case s3err.ErrEntityTooLarge:
|
||||
return "InvalidInput"
|
||||
default:
|
||||
return "IncompleteSignature"
|
||||
}
|
||||
|
||||
@@ -3041,5 +3041,6 @@ func TestIamAuthErrorCode(t *testing.T) {
|
||||
assert.Equal(t, "SignatureDoesNotMatch", iamAuthErrorCode(s3err.ErrSignatureDoesNotMatch))
|
||||
assert.Equal(t, "AccessDenied", iamAuthErrorCode(s3err.ErrAccessDenied))
|
||||
assert.Equal(t, "RequestTimeTooSkewed", iamAuthErrorCode(s3err.ErrRequestTimeTooSkewed))
|
||||
assert.Equal(t, "InvalidInput", iamAuthErrorCode(s3err.ErrEntityTooLarge))
|
||||
assert.Equal(t, "IncompleteSignature", iamAuthErrorCode(s3err.ErrMissingFields))
|
||||
}
|
||||
@@ -1106,9 +1106,9 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
|
||||
// GetBucketLifecycleConfiguration
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLifecycleConfigurationHandler, ACTION_READ)), "GET")).Queries("lifecycle", "")
|
||||
// PutBucketLifecycleConfiguration
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketLifecycleConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("lifecycle", "")
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketLifecycleConfigurationHandler, ACTION_ADMIN)), "PUT")).Queries("lifecycle", "")
|
||||
// DeleteBucketLifecycleConfiguration
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketLifecycleHandler, ACTION_WRITE)), "DELETE")).Queries("lifecycle", "")
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketLifecycleHandler, ACTION_ADMIN)), "DELETE")).Queries("lifecycle", "")
|
||||
|
||||
// GetBucketLocation
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLocationHandler, ACTION_READ)), "GET")).Queries("location", "")
|
||||
@@ -1500,6 +1500,21 @@ func (s3a *S3ApiServer) AuthenticateRequest(r *http.Request) (string, interface{
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
// FilerSigningKey is the current jwt.filer_signing.key, which follows a
|
||||
// security.toml reload.
|
||||
func (s3a *S3ApiServer) FilerSigningKey() security.SigningKey {
|
||||
if s3a.filerGuard == nil {
|
||||
return nil
|
||||
}
|
||||
return s3a.filerGuard.SigningKey()
|
||||
}
|
||||
|
||||
// AuthEnabled reports whether requests must carry valid credentials. With no
|
||||
// identities configured, every request is allowed as the admin.
|
||||
func (s3a *S3ApiServer) AuthEnabled() bool {
|
||||
return s3a.iam == nil || s3a.iam.isEnabled()
|
||||
}
|
||||
|
||||
// DefaultAllow returns whether access is allowed by default when no policy is found
|
||||
func (s3a *S3ApiServer) DefaultAllow() bool {
|
||||
if s3a.iam == nil || s3a.iam.iamIntegration == nil {
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -529,3 +531,60 @@ func TestRouting_BucketRootPreflightUsesBucketCORS(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Write covers a bucket's objects, not its configuration: an identity holding
|
||||
// Write on the bucket is refused lifecycle writes, while Admin on the bucket or
|
||||
// a policy granting s3:PutLifecycleConfiguration passes. The bucket is
|
||||
// negatively cached, so a request that gets past authorization ends at the
|
||||
// handler's NoSuchBucket.
|
||||
func TestRouting_BucketLifecycleWritesNeedAdmin(t *testing.T) {
|
||||
const bucket = "config-bucket"
|
||||
configFile := filepath.Join(t.TempDir(), "s3.json")
|
||||
require.NoError(t, os.WriteFile(configFile, []byte(`{
|
||||
"identities": [
|
||||
{"name":"writer","credentials":[{"accessKey":"WRITERKEY","secretKey":"writersecret"}],"actions":["Read:`+bucket+`","Write:`+bucket+`","List:`+bucket+`","Tagging:`+bucket+`"]},
|
||||
{"name":"bucketadmin","credentials":[{"accessKey":"BUCKETADMINKEY","secretKey":"bucketadminsecret"}],"actions":["Admin:`+bucket+`"]},
|
||||
{"name":"lifecycler","credentials":[{"accessKey":"LIFECYCLERKEY","secretKey":"lifecyclersecret"}],"policyNames":["LifecyclePolicy"]}
|
||||
],
|
||||
"policies": [
|
||||
{"name":"LifecyclePolicy","content":"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:PutLifecycleConfiguration\"],\"Resource\":[\"arn:aws:s3:::`+bucket+`\"]}]}"}
|
||||
]
|
||||
}`), 0600))
|
||||
|
||||
s3a := setupRoutingTestServer(t)
|
||||
s3a.iam = NewIdentityAccessManagementWithStore(&S3ApiServerOption{Config: configFile}, nil, "memory")
|
||||
require.True(t, s3a.iam.isEnabled())
|
||||
s3a.cb = &CircuitBreaker{s3a: s3a}
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.SetNegativeCache(bucket)
|
||||
router := mux.NewRouter()
|
||||
s3a.registerRouter(router)
|
||||
|
||||
send := func(accessKey, secretKey, method, body string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(method, "http://localhost:8333/"+bucket+"?lifecycle", strings.NewReader(body))
|
||||
creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
|
||||
_, err := v4.NewSigner(creds).Sign(req, strings.NewReader(body), "s3", "us-east-1", time.Now())
|
||||
require.NoError(t, err)
|
||||
rr := httptest.NewRecorder()
|
||||
router.ServeHTTP(rr, req)
|
||||
return rr
|
||||
}
|
||||
|
||||
const lifecycleXML = `<LifecycleConfiguration><Rule><ID>r</ID><Status>Enabled</Status><Filter><Prefix>p/</Prefix></Filter><Expiration><Days>1</Days></Expiration></Rule></LifecycleConfiguration>`
|
||||
for _, tc := range []struct {
|
||||
method, body string
|
||||
}{
|
||||
{http.MethodPut, lifecycleXML},
|
||||
{http.MethodDelete, ""},
|
||||
} {
|
||||
rr := send("WRITERKEY", "writersecret", tc.method, tc.body)
|
||||
assert.Equal(t, http.StatusForbidden, rr.Code, "writer %s", tc.method)
|
||||
assert.Contains(t, rr.Body.String(), "AccessDenied", "writer %s", tc.method)
|
||||
|
||||
for _, key := range [][2]string{{"BUCKETADMINKEY", "bucketadminsecret"}, {"LIFECYCLERKEY", "lifecyclersecret"}} {
|
||||
rr = send(key[0], key[1], tc.method, tc.body)
|
||||
assert.Equal(t, http.StatusNotFound, rr.Code, "%s %s", key[0], tc.method)
|
||||
assert.Contains(t, rr.Body.String(), "NoSuchBucket", "%s %s", key[0], tc.method)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -58,6 +58,11 @@ func (st *S3TablesApiServer) SetLocalAccountChecker(fn func(string) bool) {
|
||||
st.handler.SetLocalAccountChecker(fn)
|
||||
}
|
||||
|
||||
// SetEncryptionChecker sets the handler's SSE-S3 availability predicate.
|
||||
func (st *S3TablesApiServer) SetEncryptionChecker(fn func() bool) {
|
||||
st.handler.SetEncryptionChecker(fn)
|
||||
}
|
||||
|
||||
// S3TablesHandler handles S3 Tables API requests
|
||||
func (st *S3TablesApiServer) S3TablesHandler(w http.ResponseWriter, r *http.Request) {
|
||||
st.handler.HandleRequest(w, r, st)
|
||||
@@ -72,6 +77,7 @@ func (st *S3TablesApiServer) WithFilerClient(streamingMode bool, fn func(filer_p
|
||||
func (s3a *S3ApiServer) registerS3TablesRoutes(router *mux.Router) {
|
||||
// Create S3 Tables handler
|
||||
s3TablesApi := NewS3TablesApiServer(s3a)
|
||||
s3TablesApi.SetEncryptionChecker(GetSSES3KeyManager().IsConfigured)
|
||||
if s3a.iam != nil {
|
||||
s3TablesApi.SetLocalAccountChecker(s3a.iam.isConfiguredAccountID)
|
||||
}
|
||||
|
||||
@@ -159,6 +159,34 @@ func TestLanceAuthRejectsFailedSignature(t *testing.T) {
|
||||
assert.NotEqual(t, http.StatusOK, rr.Code)
|
||||
}
|
||||
|
||||
// The Lance catalog takes any x-api-key while the S3 port runs without
|
||||
// identities, as the admin, and refuses an unknown key once auth is on.
|
||||
func TestLanceAuthTakesAnyKeyOnlyWhileS3AuthIsOff(t *testing.T) {
|
||||
s3a := &S3ApiServer{iam: &IdentityAccessManagement{}}
|
||||
server := lance.NewServer(nil, s3a)
|
||||
server.SetCredentialValidator(s3a)
|
||||
|
||||
call := func() (int, string) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/namespace/list", nil)
|
||||
req.Header.Set("x-api-key", "unknown:key")
|
||||
identity := ""
|
||||
rr := httptest.NewRecorder()
|
||||
server.Auth(func(w http.ResponseWriter, r *http.Request) {
|
||||
identity = s3_constants.GetIdentityNameFromContext(r)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
})(rr, req)
|
||||
return rr.Code, identity
|
||||
}
|
||||
|
||||
code, identity := call()
|
||||
assert.Equal(t, http.StatusOK, code)
|
||||
assert.Equal(t, s3_constants.AccountAdminId, identity)
|
||||
|
||||
s3a.iam.isAuthEnabled = true
|
||||
code, _ = call()
|
||||
assert.Equal(t, http.StatusUnauthorized, code)
|
||||
}
|
||||
|
||||
func TestAuthSignatureOnlyScrubsAccountHeader(t *testing.T) {
|
||||
s3a := setupDefaultAllowAuthServer(t)
|
||||
|
||||
|
||||
@@ -11,8 +11,10 @@ import (
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
@@ -107,6 +109,20 @@ func (h *S3TablesHandler) mutateEntry(
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
path string,
|
||||
mutate func(entry *filer_pb.Entry) error,
|
||||
) error {
|
||||
return h.mutateEntryAsserting(ctx, client, path, nil, mutate)
|
||||
}
|
||||
|
||||
// mutateEntryAsserting is mutateEntry with the keys in absent asserted at the
|
||||
// write too, as this package's own attributes are: one absent when the entry was
|
||||
// read and written by someone else since fails the compare, instead of being
|
||||
// written over with the entry as it was read.
|
||||
func (h *S3TablesHandler) mutateEntryAsserting(
|
||||
ctx context.Context,
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
path string,
|
||||
absent []string,
|
||||
mutate func(entry *filer_pb.Entry) error,
|
||||
) error {
|
||||
dir, name := splitPath(path)
|
||||
|
||||
@@ -120,7 +136,7 @@ func (h *S3TablesHandler) mutateEntry(
|
||||
}
|
||||
|
||||
entry := resp.Entry
|
||||
expected := snapshotExtended(entry.Extended)
|
||||
expected := snapshotExtended(entry.Extended, absent...)
|
||||
if entry.Extended == nil {
|
||||
entry.Extended = make(map[string][]byte)
|
||||
}
|
||||
@@ -401,13 +417,25 @@ func (h *S3TablesHandler) getExtendedAttribute(ctx context.Context, client filer
|
||||
|
||||
// readNamespaceMetadataAttr returns the namespace-metadata xattr at
|
||||
// namespacePath. A directory without it is not a namespace — a delete that kept
|
||||
// the data under it leaves one — so its absence reads as a missing entry.
|
||||
// the data under it leaves one — so its absence reads as a missing entry. So
|
||||
// does a namespace still marked by its create: like a table's, it is absent to
|
||||
// every reader until the create confirms it.
|
||||
func (h *S3TablesHandler) readNamespaceMetadataAttr(ctx context.Context, client filer_pb.SeaweedFilerClient, namespacePath string) ([]byte, error) {
|
||||
data, err := h.getExtendedAttribute(ctx, client, namespacePath, ExtendedKeyMetadata)
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
entry, err := h.lookupEntry(ctx, client, namespacePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, ok := entry.Extended[ExtendedKeyMetadata]
|
||||
if !ok || isUnfinishedCreate(entry.Extended) {
|
||||
return nil, fmt.Errorf("%w: %s", filer_pb.ErrNotFound, namespacePath)
|
||||
}
|
||||
return data, err
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// isFinishedNamespaceEntry is isNamespaceEntry for a namespace its create has
|
||||
// confirmed, the only kind a reader or a delete may act on.
|
||||
func isFinishedNamespaceEntry(entry *filer_pb.Entry) bool {
|
||||
return isNamespaceEntry(entry) && !isUnfinishedCreate(entry.Extended)
|
||||
}
|
||||
|
||||
// loadNamespaceMetadata resolves a namespace to its metadata. A directory that
|
||||
@@ -455,14 +483,6 @@ func (h *S3TablesHandler) lookupEntry(ctx context.Context, client filer_pb.Seawe
|
||||
return resp.Entry, nil
|
||||
}
|
||||
|
||||
// deleteExtendedAttribute deletes an extended attribute from an entry
|
||||
func (h *S3TablesHandler) deleteExtendedAttribute(ctx context.Context, client filer_pb.SeaweedFilerClient, path, key string) error {
|
||||
return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
|
||||
delete(extended, key)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// deleteDirectory deletes a directory and all its contents
|
||||
func (h *S3TablesHandler) deleteDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error {
|
||||
dir, name := splitPath(path)
|
||||
@@ -499,9 +519,46 @@ func (h *S3TablesHandler) deleteDirectoryIfMatches(ctx context.Context, client f
|
||||
return nil
|
||||
}
|
||||
|
||||
// errDirNotEmpty is a directory that holds something the caller meant to leave
|
||||
// alone: a file in a leftover tree, or an entry the filer found in a directory
|
||||
// it was asked to delete without its contents.
|
||||
var errDirNotEmpty = errors.New("directory is not empty")
|
||||
|
||||
// deleteEmptyDirectoryIfMatches deletes a directory without its contents, only
|
||||
// while it still carries the extended attributes the caller captured. The
|
||||
// filer checks both under the path lock, in the one delete: a changed
|
||||
// attribute is ErrConcurrentUpdate, and an entry that appeared in the
|
||||
// directory is errDirNotEmpty, so neither a directory taken back since the
|
||||
// caller looked at it nor a file written into it since is lost. The mark of an
|
||||
// empty S3 object stored on the directory is asserted absent too, so one
|
||||
// written since is kept the same way.
|
||||
func (h *S3TablesHandler) deleteEmptyDirectoryIfMatches(ctx context.Context, client filer_pb.SeaweedFilerClient, path string, expected *filer_pb.Entry) error {
|
||||
dir, name := splitPath(path)
|
||||
resp, err := client.DeleteEntry(ctx, &filer_pb.DeleteEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
IsDeleteData: true,
|
||||
ExpectedExtended: snapshotExtended(expected.Extended, s3_constants.SeaweedFSPrefixObject),
|
||||
})
|
||||
if status.Code(err) == codes.FailedPrecondition {
|
||||
return fmt.Errorf("%w: %s", ErrConcurrentUpdate, path)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp != nil && resp.Error != "" {
|
||||
if err := filer.DeleteEntryError(resp.Error); !errors.Is(err, filer.ErrNonEmptyFolder) {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("%w: %s", errDirNotEmpty, path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
errParentDeleting = errors.New("parent is being deleted")
|
||||
errParentGone = errors.New("parent was deleted")
|
||||
errParentCreating = errors.New("parent is still being created")
|
||||
errNoDeleteClaim = errors.New("claim not held")
|
||||
errBackOutFailed = errors.New("backing out the create failed")
|
||||
errSourceGone = errors.New("rename source was deleted")
|
||||
@@ -527,18 +584,71 @@ func (h *S3TablesHandler) claimForDelete(ctx context.Context, client filer_pb.Se
|
||||
return claim, claimed, nil
|
||||
}
|
||||
|
||||
// releaseDeleteClaim removes claim from path while path still holds it, even
|
||||
// after the request was cancelled.
|
||||
func (h *S3TablesHandler) releaseDeleteClaim(ctx context.Context, client filer_pb.SeaweedFilerClient, path, claim string) {
|
||||
err := h.mutateEntryExtended(context.WithoutCancel(ctx), client, path, func(extended map[string][]byte) error {
|
||||
// tryReleaseDeleteClaim removes claim from path once, while path still holds
|
||||
// it. A claim another delete replaced or lifted, or an entry that is gone,
|
||||
// leaves nothing to release and is not an error.
|
||||
func (h *S3TablesHandler) tryReleaseDeleteClaim(ctx context.Context, client filer_pb.SeaweedFilerClient, path, claim string) error {
|
||||
err := h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
|
||||
if string(extended[ExtendedKeyDeleting]) != claim {
|
||||
return errNoDeleteClaim
|
||||
}
|
||||
delete(extended, ExtendedKeyDeleting)
|
||||
return nil
|
||||
})
|
||||
if err != nil && !errors.Is(err, errNoDeleteClaim) && !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
glog.Warningf("release delete claim on %s: %v", path, err)
|
||||
if errors.Is(err, errNoDeleteClaim) || errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// releaseDeleteClaim removes claim from path while path still holds it, even
|
||||
// after the request was cancelled. A claim left behind refuses creates under a
|
||||
// bucket or namespace and makes a leftover directory read as taken, so a failed
|
||||
// release is retried, including one that lost to another write of the entry.
|
||||
func (h *S3TablesHandler) releaseDeleteClaim(ctx context.Context, client filer_pb.SeaweedFilerClient, path, claim string) {
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
var err error
|
||||
for attempt := 0; attempt < extendedAttributeUpdateAttempts; attempt++ {
|
||||
if attempt > 0 {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
if err = h.tryReleaseDeleteClaim(ctx, client, path, claim); err == nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
glog.Warningf("release delete claim on %s: %v", path, err)
|
||||
}
|
||||
|
||||
// isTableBucketCreatedAt accepts only the table bucket created at createdAt. A
|
||||
// bucket deleted and created again under the same name has a CreatedAt of its
|
||||
// own, so a create authorized against the first reads the second as gone.
|
||||
func isTableBucketCreatedAt(createdAt time.Time) func(*filer_pb.Entry) bool {
|
||||
return func(entry *filer_pb.Entry) bool {
|
||||
if !IsTableBucketEntry(entry) {
|
||||
return false
|
||||
}
|
||||
var metadata tableBucketMetadata
|
||||
if json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata) != nil {
|
||||
return false
|
||||
}
|
||||
return metadata.CreatedAt.Equal(createdAt)
|
||||
}
|
||||
}
|
||||
|
||||
// isNamespaceCreatedAt is isTableBucketCreatedAt for a namespace its create has
|
||||
// confirmed: one deleted and created again under the same name reads as the
|
||||
// authorized one gone. A bucket created again holds only namespaces of its own,
|
||||
// so a create bound to its namespace this way is bound to the bucket too.
|
||||
func isNamespaceCreatedAt(createdAt time.Time) func(*filer_pb.Entry) bool {
|
||||
return func(entry *filer_pb.Entry) bool {
|
||||
if !isFinishedNamespaceEntry(entry) {
|
||||
return false
|
||||
}
|
||||
var metadata namespaceMetadata
|
||||
if json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata) != nil {
|
||||
return false
|
||||
}
|
||||
return metadata.CreatedAt.Equal(createdAt)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -549,7 +659,8 @@ func isNamespaceEntry(entry *filer_pb.Entry) bool {
|
||||
}
|
||||
|
||||
// checkParentOpen returns errParentGone if parentPath is missing or not a
|
||||
// catalog entry, and errParentDeleting if a delete has claimed it.
|
||||
// catalog entry, errParentCreating if the create that wrote it has not
|
||||
// confirmed it yet, and errParentDeleting if a delete has claimed it.
|
||||
func (h *S3TablesHandler) checkParentOpen(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool) error {
|
||||
_, err := h.openParent(ctx, client, parentPath, isParent)
|
||||
return err
|
||||
@@ -568,6 +679,9 @@ func (h *S3TablesHandler) openParent(ctx context.Context, client filer_pb.Seawee
|
||||
if !isParent(entry) {
|
||||
return nil, errParentGone
|
||||
}
|
||||
if isUnfinishedCreate(entry.Extended) {
|
||||
return nil, errParentCreating
|
||||
}
|
||||
if _, claimed := entry.Extended[ExtendedKeyDeleting]; claimed {
|
||||
return nil, errParentDeleting
|
||||
}
|
||||
@@ -636,6 +750,28 @@ func (h *S3TablesHandler) mutateCatalogRecord(
|
||||
})
|
||||
}
|
||||
|
||||
// requireTableCreatedAt refuses a write to the record at path unless it is the
|
||||
// table created at createdAt, the one the caller was authorized against: a
|
||||
// table deleted and created again under the name has a CreatedAt of its own.
|
||||
// Only the creation time is compared, so the commits an Iceberg writer lands
|
||||
// meanwhile do not turn a write of the table's policy, tags or maintenance
|
||||
// settings into a conflict.
|
||||
func requireTableCreatedAt(extended map[string][]byte, path string, createdAt time.Time) error {
|
||||
var current tableMetadataInternal
|
||||
if json.Unmarshal(extended[ExtendedKeyMetadata], ¤t) != nil || !current.CreatedAt.Equal(createdAt) {
|
||||
return fmt.Errorf("%w: %s holds another table", ErrConcurrentUpdate, path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// requireTableBucketCreatedAt is requireTableCreatedAt for a table bucket.
|
||||
func requireTableBucketCreatedAt(extended map[string][]byte, path string, createdAt time.Time) error {
|
||||
if !isTableBucketCreatedAt(createdAt)(&filer_pb.Entry{Extended: extended}) {
|
||||
return fmt.Errorf("%w: %s holds another table bucket", ErrConcurrentUpdate, path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// confirmParent re-checks the parent after a create wrote written onto
|
||||
// childPath. If the parent is open, it clears the record's unfinished mark —
|
||||
// then checks the parent once more, since a delete claiming it in between
|
||||
@@ -644,6 +780,20 @@ func (h *S3TablesHandler) mutateCatalogRecord(
|
||||
// it removes what the create wrote and returns why, or errBackOutFailed if
|
||||
// the record stays.
|
||||
func (h *S3TablesHandler) confirmParent(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool, childPath string, written map[string][]byte, emptyDirs ...string) error {
|
||||
return h.confirmChild(ctx, client, parentPath, isParent, true, childPath, written, emptyDirs)
|
||||
}
|
||||
|
||||
// confirmParentOnce is confirmParent for a record that other creates build on,
|
||||
// a namespace. Checking the parent again after the mark comes off would leave a
|
||||
// window in which a child is created in the record and the check then removes
|
||||
// the record under it, so the parent is checked once, before the mark comes
|
||||
// off. That is enough while a delete of the parent counts the marked record: one
|
||||
// claiming the parent after the check scans the record and is refused.
|
||||
func (h *S3TablesHandler) confirmParentOnce(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool, childPath string, written map[string][]byte, emptyDirs ...string) error {
|
||||
return h.confirmChild(ctx, client, parentPath, isParent, false, childPath, written, emptyDirs)
|
||||
}
|
||||
|
||||
func (h *S3TablesHandler) confirmChild(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool, recheck bool, childPath string, written map[string][]byte, emptyDirs []string) error {
|
||||
parent, err := h.openParent(ctx, client, parentPath, isParent)
|
||||
confirmed := false
|
||||
if err == nil {
|
||||
@@ -651,7 +801,9 @@ func (h *S3TablesHandler) confirmParent(ctx context.Context, client filer_pb.Sea
|
||||
// create wrote; a failed write backs the record out like a closed parent.
|
||||
if err = h.removeExtendedAttributesIf(ctx, client, childPath, written, ExtendedKeyDeleting); err == nil {
|
||||
confirmed = true
|
||||
err = h.parentStillOpen(ctx, client, parentPath, isParent, parent)
|
||||
if recheck {
|
||||
err = h.parentStillOpen(ctx, client, parentPath, isParent, parent)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
@@ -753,6 +905,159 @@ func (h *S3TablesHandler) dirHoldsFiles(ctx context.Context, client filer_pb.Sea
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// claimTreeDirectory writes a fresh delete claim onto the directory at dirPath
|
||||
// while it still carries expected, and returns the claim with the attributes it
|
||||
// left. A claim seen in expected, one a delete that died left, is replaced like
|
||||
// any other attribute; one written since fails the compare.
|
||||
//
|
||||
// The claim is what ties the later delete to this directory: the delete
|
||||
// requires it, and only this directory carries it. A file or an S3 object at
|
||||
// dirPath is refused with errDirNotEmpty, whether it was there when the entry
|
||||
// was read or took the directory's place before the write, which the filer
|
||||
// refuses since it never writes a directory entry over a file. One that takes
|
||||
// the place after the write lacks the claim, so the delete is refused instead.
|
||||
// An S3 object landing on the directory between the read and the write carries
|
||||
// the prefix-object mark, which the write asserts absent, so the write fails
|
||||
// instead of putting the bare directory back over the object.
|
||||
func (h *S3TablesHandler) claimTreeDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, dirPath string, expected map[string][]byte) (string, map[string][]byte, error) {
|
||||
claim := generateVersionToken()
|
||||
var claimed map[string][]byte
|
||||
err := h.mutateEntryAsserting(ctx, client, dirPath, []string{s3_constants.SeaweedFSPrefixObject}, func(entry *filer_pb.Entry) error {
|
||||
if !entry.IsDirectory || entry.IsDirectoryKeyObject() {
|
||||
return fmt.Errorf("%w: %s is an object", errDirNotEmpty, dirPath)
|
||||
}
|
||||
for key, want := range expected {
|
||||
if !bytes.Equal(entry.Extended[key], want) {
|
||||
return fmt.Errorf("%w: %s changed on %s", ErrConcurrentUpdate, key, dirPath)
|
||||
}
|
||||
}
|
||||
entry.Extended[ExtendedKeyDeleting] = []byte(claim)
|
||||
claimed = snapshotExtended(entry.Extended)
|
||||
return nil
|
||||
})
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return "", nil, fmt.Errorf("%w: %s", ErrConcurrentUpdate, dirPath)
|
||||
}
|
||||
if err != nil && !errors.Is(err, errDirNotEmpty) && !errors.Is(err, ErrConcurrentUpdate) {
|
||||
// The filer reports the refused write over a file only as an opaque
|
||||
// failure, so look at what holds the path now.
|
||||
if current, lookupErr := h.lookupEntry(ctx, client, dirPath); lookupErr == nil && !current.IsDirectory {
|
||||
return "", nil, fmt.Errorf("%w: %s is an object", errDirNotEmpty, dirPath)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return claim, claimed, nil
|
||||
}
|
||||
|
||||
// removeTreeDirectory removes a directory the walk listed as listed, under a
|
||||
// claim of its own, and lifts the claim if the delete is refused.
|
||||
func (h *S3TablesHandler) removeTreeDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, dirPath string, listed *filer_pb.Entry) error {
|
||||
claim, claimed, err := h.claimTreeDirectory(ctx, client, dirPath, snapshotExtended(listed.Extended))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := h.deleteEmptyDirectoryIfMatches(ctx, client, dirPath, &filer_pb.Entry{Extended: claimed}); err != nil {
|
||||
h.releaseDeleteClaim(ctx, client, dirPath, claim)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeEmptyTree removes dirPath and every directory under it, provided no
|
||||
// file, no S3 object stored on a directory entry, and no catalog record sits
|
||||
// anywhere in the tree, dirPath included. dirEntry is dirPath as the caller saw
|
||||
// it.
|
||||
//
|
||||
// dirPath is claimed first, in a write that requires the attributes the caller
|
||||
// saw: a directory taken back as a namespace since then fails it, before
|
||||
// anything under it is touched, and the claim keeps it from being taken back
|
||||
// while the tree goes. With dirPath no namespace, nothing under it can become a
|
||||
// catalog record either, so a record the walk meets was there before and is
|
||||
// refused, not removed.
|
||||
//
|
||||
// The walk lists the tree a page at a time and stops at the first file, object
|
||||
// or record with errDirNotEmpty, before anything is removed. Directories then go
|
||||
// deepest first, each claimed with the attributes it had when listed and
|
||||
// deleted under that claim, which refuses a directory that holds an entry by
|
||||
// now or was replaced by a file. dirPath goes last, still under its claim; any
|
||||
// other way out lifts the claim.
|
||||
func (h *S3TablesHandler) removeEmptyTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dirPath string, dirEntry *filer_pb.Entry) (err error) {
|
||||
claim, claimed, err := h.claimTreeDirectory(ctx, client, dirPath, snapshotExtended(dirEntry.Extended))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
h.releaseDeleteClaim(ctx, client, dirPath, claim)
|
||||
}
|
||||
}()
|
||||
|
||||
type listedDir struct {
|
||||
path string
|
||||
entry *filer_pb.Entry
|
||||
}
|
||||
listed := []listedDir{{dirPath, &filer_pb.Entry{Extended: claimed}}}
|
||||
|
||||
walkCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
// A directory is listed after its parent, so removing in reverse order
|
||||
// removes every directory after the ones under it.
|
||||
for next := 0; next < len(listed); next++ {
|
||||
dir := listed[next].path
|
||||
lastFileName := ""
|
||||
for {
|
||||
resp, err := client.ListEntries(walkCtx, &filer_pb.ListEntriesRequest{
|
||||
Directory: dir,
|
||||
Limit: dirListPageSize,
|
||||
StartFromFileName: lastFileName,
|
||||
InclusiveStartFrom: lastFileName == "",
|
||||
})
|
||||
if err != nil {
|
||||
if status.Code(err) == codes.NotFound {
|
||||
break // gone, so its delete below reports the change
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
received := false
|
||||
for {
|
||||
entry, err := resp.Recv()
|
||||
if err != nil {
|
||||
if err == io.EOF || status.Code(err) == codes.NotFound {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
if entry.Entry == nil {
|
||||
continue
|
||||
}
|
||||
received = true
|
||||
lastFileName = entry.Entry.Name
|
||||
child := path.Join(dir, entry.Entry.Name)
|
||||
if !entry.Entry.IsDirectory || entry.Entry.IsDirectoryKeyObject() {
|
||||
return fmt.Errorf("%w: %s is an object", errDirNotEmpty, child)
|
||||
}
|
||||
if _, isRecord := entry.Entry.Extended[ExtendedKeyMetadata]; isRecord {
|
||||
return fmt.Errorf("%w: %s is a catalog record", errDirNotEmpty, child)
|
||||
}
|
||||
listed = append(listed, listedDir{child, entry.Entry})
|
||||
}
|
||||
if !received {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for i := len(listed) - 1; i > 0; i-- {
|
||||
if err := h.removeTreeDirectory(ctx, client, listed[i].path, listed[i].entry); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return h.deleteEmptyDirectoryIfMatches(ctx, client, dirPath, &filer_pb.Entry{Extended: claimed})
|
||||
}
|
||||
|
||||
// keepAsLeftover clears a namespace's catalog attributes and the delete's
|
||||
// claim in one write, while the entry still holds claimed, keeping the
|
||||
// directory and the data under it. The write compares against claimed so the
|
||||
|
||||
Loaded 100 of 155 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user