Merge branch 'master' into fix-delete-offline-replica

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 committed 2026-11-02 11:48:09 -08:00
commit 64f92bfa8a
155 files changed
+12108 -2073

No files matched your search

@@ -0,0 +1,37 @@
name: "Cancel runs of closed PRs"
# Merging or closing a PR leaves its pull_request runs going: concurrency
# groups cancel a run only when a newer push arrives. pull_request_target gives
# this job a token that can cancel runs, fork PRs included. It checks out and
# runs no PR code.
on:
pull_request_target:
types: [closed]
permissions:
actions: write
pull-requests: read
jobs:
cancel:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Cancel the PR's queued and running pull_request runs
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
PR: ${{ github.event.pull_request.number }}
HEAD_REF: ${{ github.event.pull_request.head.ref }}
run: |
set -euo pipefail
# Match the PR's own commits: forks reuse branch names such as master.
shas=$(gh api --paginate "repos/$GH_REPO/pulls/$PR/commits" --jq '.[].sha')
for status in queued in_progress waiting pending requested; do
gh run list --branch "$HEAD_REF" --event pull_request --status "$status" \
--limit 200 --json databaseId,headSha --jq '.[] | "\(.databaseId) \(.headSha)"'
done | while read -r id sha; do
grep -qx "$sha" <<<"$shas" || continue
echo "cancel run $id at $sha"
gh run cancel "$id" || true
done
+19 -1
View File
@@ -8,6 +8,7 @@ on:
- 'seaweed-volume/**'
- 'seaweed-common/**'
- 'seaweed-worker/**'
- 'seaweed-vector/**'
- 'docker/**'
- 'go.mod'
- 'go.sum'
@@ -50,7 +51,8 @@ jobs:
~/.cargo/git
seaweed-volume/target
seaweed-worker/target/${{ matrix.target }}/release
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
seaweed-vector/target/${{ matrix.target }}/release
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }}
restore-keys: |
rust-docker-dev-${{ matrix.target }}-
@@ -83,6 +85,12 @@ jobs:
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
- name: Build the vector engine
run: |
cd seaweed-vector
cargo build --release -p weed-vector-server --target ${{ matrix.target }}
cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }}
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
@@ -90,6 +98,7 @@ jobs:
path: |
weed-volume-normal-${{ matrix.arch }}
weed-worker-${{ matrix.arch }}
weed-vector-${{ matrix.arch }}
build-dev-containers:
needs: [build-rust-binaries]
@@ -127,8 +136,17 @@ jobs:
echo "Placed pre-built Rust worker for ${arch}"
fi
done
mkdir -p docker/weed-vector-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-vector-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
echo "Placed pre-built vector engine for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
ls -la docker/weed-worker-prebuilt/
ls -la docker/weed-vector-prebuilt/
- name: Docker meta
id: docker_meta
+32 -1
View File
@@ -96,7 +96,8 @@ jobs:
~/.cargo/git
seaweed-volume/target
seaweed-worker/target/${{ matrix.target }}/release
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
seaweed-vector/target/${{ matrix.target }}/release
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }}
restore-keys: |
rust-docker-${{ matrix.target }}-
@@ -137,6 +138,17 @@ jobs:
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
- name: Build the vector engine
run: |
# An older source_ref predates the engine; its image goes without one.
if [ ! -f seaweed-vector/crates/server/Cargo.toml ]; then
echo "No vector engine in this source ref"
exit 0
fi
cd seaweed-vector
cargo build --release -p weed-vector-server --target ${{ matrix.target }}
cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }}
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
@@ -145,6 +157,7 @@ jobs:
weed-volume-large-disk-${{ matrix.arch }}
weed-volume-normal-${{ matrix.arch }}
weed-worker-${{ matrix.arch }}
weed-vector-${{ matrix.arch }}
build:
needs: [setup, build-rust-binaries]
@@ -217,8 +230,17 @@ jobs:
echo "Placed pre-built Rust worker for ${arch}"
fi
done
mkdir -p docker/weed-vector-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-vector-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
echo "Placed pre-built vector engine for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
ls -la docker/weed-worker-prebuilt/
ls -la docker/weed-vector-prebuilt/
- name: Docker meta
id: docker_meta
@@ -342,8 +364,17 @@ jobs:
echo "Placed pre-built Rust worker for ${arch}"
fi
done
mkdir -p docker/weed-vector-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-vector-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
echo "Placed pre-built vector engine for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
ls -la docker/weed-worker-prebuilt/
ls -la docker/weed-vector-prebuilt/
- name: Create BuildKit config for local scan build
if: needs.setup.outputs.publish != 'true'
run: |
@@ -43,9 +43,9 @@ concurrency:
jobs:
# ── Pre-build the Rust binaries natively ────────────────────────────
# The volume server and the Rust maintenance worker, cross-compiled for
# amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into
# ~15 minutes of native compilation.
# The volume server, the Rust maintenance worker and the vector engine,
# cross-compiled for amd64 and arm64 without QEMU, turning a 5-hour emulated
# cargo build into ~15 minutes of native compilation.
build-rust-binaries:
runs-on: ubuntu-24.04
strategy:
@@ -73,7 +73,8 @@ jobs:
~/.cargo/git
seaweed-volume/target
seaweed-worker/target/${{ matrix.target }}/release
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
seaweed-vector/target/${{ matrix.target }}/release
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }}
restore-keys: |
rust-docker-${{ matrix.target }}-
@@ -114,6 +115,12 @@ jobs:
cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
- name: Build the vector engine
run: |
cd seaweed-vector
cargo build --release -p weed-vector-server --target ${{ matrix.target }}
cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }}
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
@@ -122,6 +129,7 @@ jobs:
weed-volume-large-disk-${{ matrix.arch }}
weed-volume-normal-${{ matrix.arch }}
weed-worker-${{ matrix.arch }}
weed-vector-${{ matrix.arch }}
# One job per (variant, platform) on a native runner, pushed by digest;
# the merge job stitches the digests into one multi-arch tag.
@@ -198,8 +206,17 @@ jobs:
echo "Placed pre-built Rust worker for ${arch}"
fi
done
mkdir -p docker/weed-vector-prebuilt
for arch in amd64 arm64; do
src="./rust-bins/weed-vector-${arch}"
if [ -f "$src" ]; then
cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
echo "Placed pre-built vector engine for ${arch}"
fi
done
ls -la docker/weed-volume-prebuilt/
ls -la docker/weed-worker-prebuilt/
ls -la docker/weed-vector-prebuilt/
- name: Free Disk Space
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
-22
View File
@@ -68,19 +68,6 @@ jobs:
restore-keys: |
rust-vector-${{ steps.toolchain.outputs.fingerprint }}-
# lance's build scripts compile their own protos and look for a protoc.
# Point them at the one protoc-bin-vendored ships, so no job depends on a
# system package and every build sees the same version.
- name: Use the vendored protoc
run: |
cd seaweed-vector
cargo fetch
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
- name: Build the vector crates
run: cd seaweed-vector && cargo build --release
@@ -125,15 +112,6 @@ jobs:
key: vectors-e2e-${{ runner.os }}-${{ runner.arch }}-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-vector/Cargo.lock', 'seaweed-volume/Cargo.lock') }}
restore-keys: |
vectors-e2e-${{ runner.os }}-${{ runner.arch }}-${{ steps.toolchain.outputs.fingerprint }}-
- name: Use the vendored protoc
run: |
cd seaweed-vector
cargo fetch
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
- name: Test Go vector gateway
run: |
go test ./weed/s3api/s3vectors -count=1
@@ -188,6 +188,83 @@ jobs:
weed-worker_${{ matrix.asset_suffix }}.tar.gz
weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5
# The vector engine behind the S3 Vectors API: Linux only, like the worker,
# whose lance dependency tree it shares.
build-rust-vector-linux:
permissions:
contents: write
runs-on: ubuntu-24.04
strategy:
matrix:
include:
- target: x86_64-unknown-linux-musl
asset_suffix: linux_amd64
- target: aarch64-unknown-linux-musl
asset_suffix: linux_arm64
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- name: Set up the musl target
uses: ./.github/actions/rust-musl-target
with:
target: ${{ matrix.target }}
- name: Cache cargo registry and target
if: runner.environment == 'github-hosted'
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
seaweed-vector/target/${{ matrix.target }}/release
key: rust-vector-release-${{ matrix.target }}-${{ hashFiles('seaweed-vector/Cargo.lock') }}
restore-keys: |
rust-vector-release-${{ matrix.target }}-
- name: Use the vendored protoc
run: |
cd seaweed-vector
cargo fetch
version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
echo "PROTOC=$protoc" >> "$GITHUB_ENV"
- name: Build the vector engine
run: |
cd seaweed-vector
cargo build --release -p weed-vector-server --target ${{ matrix.target }}
- name: Package binary
run: |
cp seaweed-vector/target/${{ matrix.target }}/release/weed-vector weed-vector
tar czf weed-vector_${{ matrix.asset_suffix }}.tar.gz weed-vector
rm weed-vector
md5sum weed-vector_${{ matrix.asset_suffix }}.tar.gz > weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5
- name: Upload release assets
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v3
with:
files: |
weed-vector_${{ matrix.asset_suffix }}.tar.gz
weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Upload artifacts
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
uses: actions/upload-artifact@v7
with:
name: rust-vector-${{ matrix.asset_suffix }}
path: |
weed-vector_${{ matrix.asset_suffix }}.tar.gz
weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5
build-rust-volume-darwin:
permissions:
contents: write
+12 -1
View File
@@ -33,6 +33,7 @@ ARG TARGETARCH
ARG TAGS
COPY weed-volume-prebuilt/ /prebuilt/
COPY weed-worker-prebuilt/ /prebuilt-worker/
COPY weed-vector-prebuilt/ /prebuilt-vector/
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
# seaweed-common is a path dependency of seaweed-volume that lives beside it,
# so the source build below needs it in the same relative position.
@@ -65,11 +66,19 @@ RUN if [ -f "/prebuilt-worker/weed-worker-${TARGETARCH}" ]; then \
echo "No pre-built Rust worker for ${TARGETARCH}" && \
touch /weed-worker; \
fi
# The vector engine carries the same lance tree, so it is pre-built or absent too.
RUN if [ -f "/prebuilt-vector/weed-vector-${TARGETARCH}" ]; then \
echo "Using pre-built vector engine for ${TARGETARCH}" && \
cp "/prebuilt-vector/weed-vector-${TARGETARCH}" /weed-vector; \
else \
echo "No pre-built vector engine for ${TARGETARCH}" && \
touch /weed-vector; \
fi
# Pre-built binaries arrive via GitHub Actions artifacts, which drop the
# executable bit, so the copied file is 0644 and exec fails with "Permission
# denied". Restore it (no-op for the empty placeholders, which stay size 0).
RUN chmod 0755 /weed-volume /weed-worker
RUN chmod 0755 /weed-volume /weed-worker /weed-vector
FROM alpine AS final
LABEL author="Chris Lu"
@@ -78,6 +87,8 @@ COPY --from=builder /go/bin/weed /usr/bin/
COPY --from=rust_builder /weed-volume /usr/bin/weed-volume
# Same for the Rust maintenance worker, which serves Lance table buckets
COPY --from=rust_builder /weed-worker /usr/bin/weed-worker
# And the vector engine behind the S3 Vectors API
COPY --from=rust_builder /weed-vector /usr/bin/weed-vector
RUN mkdir -p /etc/seaweedfs
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
+13
View File
@@ -100,6 +100,19 @@ case "$1" in
exec /usr/bin/weed-worker "$@"
;;
'vector')
shift
if [ ! -s /usr/bin/weed-vector ]; then
echo "Error: the vector engine is not available on this platform ($(uname -m))." >&2
exit 1
fi
# The gateway dials the engine from another container, so not loopback.
if [ -z "$WEED_VECTOR_LISTEN" ] && ! isArgPassed "-listen" "$@"; then
set -- --listen 0.0.0.0:8090 "$@"
fi
exec /usr/bin/weed-vector "$@"
;;
'server')
ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024"
if isArgPassed "-volume.max" "$@"; then
Whitespace-only changes.
+52 -2
View File
@@ -9,7 +9,7 @@
# curl -fsSL ... | bash -s -- --version 4.34 --dir /usr/local/bin
#
# Options:
# --component COMP Which binary to install: weed, volume-rust, worker-rust, all (default: weed)
# --component COMP Which binary to install: weed, volume-rust, worker-rust, vector, all (default: weed)
# --version VER Release version tag (default: latest)
# --large-disk Use large disk variant (5-byte offset, 8TB max volume)
# --dir DIR Installation directory (default: /usr/local/bin)
@@ -23,6 +23,7 @@ VERSION=""
LARGE_DISK=false
INSTALL_DIR="/usr/local/bin"
WORKER_INSTALLED=false
VECTOR_INSTALLED=false
# Colors (if terminal supports them)
if [ -t 1 ]; then
@@ -156,6 +157,12 @@ worker_asset_name() {
echo "weed-worker_${os}_${arch}.tar.gz"
}
# The vector engine behind the S3 Vectors API, built like the worker.
vector_asset_name() {
local os="$1" arch="$2"
echo "weed-vector_${os}_${arch}.tar.gz"
}
# Install a single component
install_component() {
local component="$1" os="$2" arch="$3"
@@ -264,8 +271,38 @@ install_component() {
ok "Installed weed-worker to ${INSTALL_DIR}/weed-worker"
;;
vector)
# Published for linux amd64/arm64 only, like the worker.
case "$os" in
linux) ;;
*) error "The vector engine is not available for ${os}. Supported: linux" ;;
esac
case "$arch" in
amd64|arm64) ;;
*) error "The vector engine is not available for ${arch}. Supported: amd64, arm64" ;;
esac
asset_name="$(vector_asset_name "$os" "$arch")"
download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
download "$download_url" "${tmpdir}/${asset_name}"
info "Extracting ${asset_name}..."
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
local vector_bin
vector_bin="$(find "$tmpdir" -name 'weed-vector' -type f | head -1)"
if [ -z "$vector_bin" ]; then
error "Could not find weed-vector binary in archive"
fi
chmod +x "$vector_bin"
install_binary "$vector_bin" "weed-vector"
VECTOR_INSTALLED=true
ok "Installed weed-vector to ${INSTALL_DIR}/weed-vector"
;;
*)
error "Unknown component: ${component}. Use: weed, volume-rust, worker-rust, all"
error "Unknown component: ${component}. Use: weed, volume-rust, worker-rust, vector, all"
;;
esac
@@ -327,6 +364,13 @@ main() {
else
install_component "worker-rust" "$os" "$arch"
fi
if [ "$os" != "linux" ] || { [ "$arch" != "amd64" ] && [ "$arch" != "arm64" ]; }; then
warn "Skipping the vector engine: no build for ${os}/${arch}"
elif ! asset_exists "https://github.com/${REPO}/releases/download/${VERSION}/$(vector_asset_name "$os" "$arch")"; then
warn "Skipping the vector engine: ${VERSION} does not carry one"
else
install_component "vector" "$os" "$arch"
fi
;;
*)
install_component "$COMPONENT" "$os" "$arch"
@@ -344,6 +388,9 @@ main() {
if [ "$WORKER_INSTALLED" = true ]; then
info " weed-worker: ${INSTALL_DIR}/weed-worker"
fi
if [ "$VECTOR_INSTALLED" = true ]; then
info " weed-vector: ${INSTALL_DIR}/weed-vector"
fi
echo ""
info "Quick start:"
info " weed master # Start master server"
@@ -352,6 +399,9 @@ main() {
if [ "$WORKER_INSTALLED" = true ]; then
info " weed-worker --admin localhost:12535 # Start the Rust maintenance worker"
fi
if [ "$VECTOR_INSTALLED" = true ]; then
info " weed-vector --help # The vector engine for S3 vector buckets"
fi
}
main
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.51"
appVersion: "4.52"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.51.0
version: 4.52.0
+1008 -1005
View File
File diff suppressed because it is too large. Load diff

Before

Width:  |  Height:  |  Size: 53 KiB

After

Width:  |  Height:  |  Size: 53 KiB

+6
View File
@@ -0,0 +1,6 @@
# lance's build scripts run $PROTOC. Point them at the protoc that
# protoc-bin-vendored ships, as this workspace's own build script does, so a
# build needs no system protoc. An explicit PROTOC still wins. Cargo reads this
# file only when it runs inside seaweed-vector/.
[env]
PROTOC = { value = ".cargo/protoc", relative = true }
+21
View File
@@ -0,0 +1,21 @@
#!/bin/sh
# Runs the protoc of the protoc-bin-vendored version in Cargo.lock, or the one
# on PATH when the registry does not hold it.
case "$(uname -s)-$(uname -m)" in
Linux-x86_64) platform=linux-x86_64 ;;
Linux-aarch64) platform=linux-aarch_64 ;;
Linux-ppc64le) platform=linux-ppcle_64 ;;
Linux-s390x) platform=linux-s390_64 ;;
Linux-i?86) platform=linux-x86_32 ;;
Darwin-x86_64) platform=macos-x86_64 ;;
Darwin-arm64) platform=macos-aarch_64 ;;
*) exec protoc "$@" ;;
esac
crate="protoc-bin-vendored-$platform"
version=$(awk -v name="\"$crate\"" '$1 == "name" && $3 == name { found = 1; next } found && $1 == "version" { gsub(/"/, "", $3); print $3; exit }' "$(dirname "$0")/../Cargo.lock")
for protoc in "${CARGO_HOME:-$HOME/.cargo}"/registry/src/*/"$crate-$version"/bin/protoc; do
if [ -x "$protoc" ]; then
exec "$protoc" "$@"
fi
done
exec protoc "$@"
+109 -18
View File
@@ -1,14 +1,16 @@
//! Lance's json_extract and json_exists parse their JSONPath again for every
//! row. A filter's path is a literal, so swap in copies that parse it once per
//! batch and return the same values.
//! batch and return the same values. json_array_contains is swapped too, and
//! compares the elements as typed JSON values: Lance's compares their text, so
//! 1 would match "1".
use std::sync::Arc;
use arrow_array::builder::{BooleanBuilder, StringBuilder};
use arrow_array::{Array, ArrayRef, LargeBinaryArray, new_null_array};
use arrow_schema::DataType;
use jsonb::RawJsonb;
use jsonb::jsonpath::{Selector, parse_json_path};
use jsonb::{OwnedJsonb, RawJsonb};
use lance::dataset::scanner::Scanner;
use lance::deps::datafusion::common::tree_node::{Transformed, TreeNode};
use lance::deps::datafusion::common::{DataFusionError, Result, ScalarValue};
@@ -34,12 +36,16 @@ fn swap(expr: Expr) -> Result<Expr> {
let op = match f.func.name() {
"json_extract" => Op::Extract,
"json_exists" => Op::Exists,
"json_array_contains" => Op::Contains,
_ => return Ok(Transformed::no(Expr::ScalarFunction(f))),
};
if !matches!(
f.args.get(1),
Some(Expr::Literal(ScalarValue::Utf8(Some(_)), _))
) {
let literal = |i: usize| {
matches!(
f.args.get(i),
Some(Expr::Literal(ScalarValue::Utf8(Some(_)), _))
)
};
if !literal(1) || (op == Op::Contains && !literal(2)) {
return Ok(Transformed::no(Expr::ScalarFunction(f)));
}
let udf = Arc::new(ScalarUDF::new_from_impl(JsonPath::new(op)));
@@ -54,6 +60,7 @@ fn swap(expr: Expr) -> Result<Expr> {
enum Op {
Extract,
Exists,
Contains,
}
#[derive(Debug, PartialEq, Eq, Hash)]
@@ -64,12 +71,13 @@ struct JsonPath {
impl JsonPath {
fn new(op: Op) -> Self {
let mut args = vec![DataType::LargeBinary, DataType::Utf8];
if op == Op::Contains {
args.push(DataType::Utf8);
}
Self {
op,
signature: Signature::exact(
vec![DataType::LargeBinary, DataType::Utf8],
Volatility::Immutable,
),
signature: Signature::exact(args, Volatility::Immutable),
}
}
}
@@ -83,6 +91,7 @@ impl ScalarUDFImpl for JsonPath {
match self.op {
Op::Extract => "json_extract",
Op::Exists => "json_exists",
Op::Contains => "json_array_contains",
}
}
@@ -93,17 +102,24 @@ impl ScalarUDFImpl for JsonPath {
fn return_type(&self, _: &[DataType]) -> Result<DataType> {
Ok(match self.op {
Op::Extract => DataType::Utf8,
Op::Exists => DataType::Boolean,
Op::Exists | Op::Contains => DataType::Boolean,
})
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
let [data, ColumnarValue::Scalar(ScalarValue::Utf8(Some(path)))] = args.args.as_slice()
else {
return Err(exec_err(format!(
"{} expects a literal JSONPath",
self.name()
)));
let (data, path, value) = match args.args.as_slice() {
[data, ColumnarValue::Scalar(ScalarValue::Utf8(Some(path)))] => (data, path, None),
[
data,
ColumnarValue::Scalar(ScalarValue::Utf8(Some(path))),
ColumnarValue::Scalar(ScalarValue::Utf8(Some(value))),
] => (data, path, Some(value)),
_ => {
return Err(exec_err(format!(
"{} expects literal arguments",
self.name()
)));
}
};
let data = data.to_array(args.number_rows)?;
let data = data
@@ -160,6 +176,37 @@ impl ScalarUDFImpl for JsonPath {
}
Arc::new(builder.finish())
}
Op::Contains => {
let value = value.map(String::as_str).unwrap_or_default();
let value: OwnedJsonb = value
.parse()
.map_err(|e| exec_err(format!("Invalid JSON value '{value}': {e}")))?;
let mut builder = BooleanBuilder::with_capacity(data.len());
for i in 0..data.len() {
if data.is_null(i) {
builder.append_null();
continue;
}
let mut selector = Selector::new(RawJsonb::new(data.value(i)));
let found = match selector.select_value(&json_path) {
Ok(Some(array)) => array
.as_raw()
.array_values()
.map_err(|e| {
exec_err(format!("Failed to read array at '{path}': {e}"))
})?
.is_some_and(|items| items.contains(&value)),
Ok(None) => false,
Err(e) => {
return Err(exec_err(format!(
"Failed to check array contains at path '{path}': {e}"
)));
}
};
builder.append_value(found);
}
Arc::new(builder.finish())
}
};
Ok(ColumnarValue::Array(out))
}
@@ -298,6 +345,48 @@ mod tests {
assert!(keys(&dataset, bad, true).await.is_err());
}
#[tokio::test]
async fn array_contains_compares_typed_values() {
let dir = TempDir::new().unwrap();
let metas = [
Some(r#"{"k": ["space", "classic"]}"#),
Some(r#"{"k": [1, 2.5]}"#),
Some(r#"{"k": ["1", "2.5"]}"#),
Some(r#"{"k": [true]}"#),
Some(r#"{"k": ["true"]}"#),
Some(r#"{"k": "space"}"#),
Some(r#"{"k": [["space"]]}"#),
Some(r#"{"k": {"a": "space"}}"#),
Some(r#"{"k": ["", "spa\"ce"]}"#),
Some(r#"{"k": []}"#),
Some(r#"{"x": 1}"#),
None,
];
let dataset = dataset(&dir, &metas).await;
for (value, want) in [
(r#""space""#, vec!["k0"]),
("1", vec!["k1"]),
("1.0", vec!["k1"]),
("2.5", vec!["k1"]),
(r#""1""#, vec!["k2"]),
("true", vec!["k3"]),
(r#""true""#, vec!["k4"]),
(r#""""#, vec!["k8"]),
(r#""spa\"ce""#, vec!["k8"]),
(r#""classic""#, vec!["k0"]),
("3", vec![]),
] {
let filter = format!(r#"json_array_contains(`metadata`, '$["k"]', '{value}')"#);
assert_eq!(
keys(&dataset, &filter, true).await.unwrap(),
want,
"{filter}"
);
}
let bad = r#"json_array_contains(`metadata`, '$["k"]', 'not json')"#;
assert!(keys(&dataset, bad, true).await.is_err());
}
#[tokio::test]
async fn apply_swaps_literal_paths_only() {
let dir = TempDir::new().unwrap();
@@ -305,7 +394,7 @@ mod tests {
let mut scanner = dataset.scan();
apply(
&mut scanner,
r#"json_extract(`metadata`, '$["k"]') = '"v"' AND json_exists(`metadata`, '$["k"]') AND json_get_string(`metadata`, 'k') = 'v'"#,
r#"json_extract(`metadata`, '$["k"]') = '"v"' AND json_exists(`metadata`, '$["k"]') AND json_get_string(`metadata`, 'k') = 'v' AND json_array_contains(`metadata`, '$["k"]', '"v"') AND NOT json_array_contains(`metadata`, '$["k"]', key)"#,
)
.unwrap();
let mut swapped = Vec::new();
@@ -325,6 +414,8 @@ mod tests {
assert_eq!(
swapped,
[
("json_array_contains".to_string(), false),
("json_array_contains".to_string(), true),
("json_exists".to_string(), true),
("json_extract".to_string(), true),
("json_get_string".to_string(), false),
+20 -7
View File
@@ -1,7 +1,8 @@
use anyhow::Result;
use anyhow::{Context, Result};
use clap::Parser;
use tonic::transport::Server;
use weed_lance_worker::catalog::NamespaceClient;
use tonic::transport::server::TcpIncoming;
use weed_lance_worker::catalog::{NamespaceClient, SigningKey};
use weed_lance_worker::dataset::FallbackOptions;
use weed_vector_server::pb::vector::vector_service_server::VectorServiceServer;
use weed_vector_server::read_cache::ReadCache;
@@ -10,15 +11,15 @@ use weed_vector_server::service::VectorServer;
/// The VectorService the S3 gateway fans vector calls out to.
#[derive(Parser)]
struct Args {
/// Lance Namespace base URL, e.g. http://localhost:8888/lance.
/// Lance Namespace base URL: the S3 gateway's Lance port (-s3.port.lance).
#[arg(
long,
env = "WEED_VECTOR_NAMESPACE",
default_value = "http://localhost:8888/lance"
default_value = "http://localhost:9101"
)]
namespace: String,
/// gRPC listen address.
#[arg(long, env = "WEED_VECTOR_LISTEN", default_value = "127.0.0.1:8080")]
#[arg(long, env = "WEED_VECTOR_LISTEN", default_value = "127.0.0.1:8090")]
listen: String,
/// object_store option fallback for deployments that vend no credentials:
/// key=value, repeatable.
@@ -31,6 +32,10 @@ struct Args {
/// Memory for caching reads of write-once table files, in MiB; 0 disables.
#[arg(long, env = "WEED_VECTOR_READ_CACHE_MB", default_value_t = 512)]
read_cache_mb: u64,
/// The `jwt.filer_signing.key` of the cluster. The Lance namespace vends
/// credentials that write index data only to a caller holding it.
#[arg(long, env = "WEED_FILER_SIGNING_KEY")]
filer_signing_key: Option<String>,
}
fn parse_option(s: &str) -> Result<(String, String), String> {
@@ -100,6 +105,11 @@ fn advertised_addr(listen: &str, masters: &[String]) -> String {
async fn main() -> Result<()> {
tracing_subscriber::fmt::init();
let args = Args::parse_from(normalize_args(std::env::args().collect()));
// Bind before registering, so the master never lists an engine that is
// about to exit because its port is taken.
let incoming = TcpIncoming::bind(args.listen.parse()?)
.with_context(|| format!("listen on {}", args.listen))?
.with_nodelay(Some(true));
if let Some(masters) = &args.masters {
let masters: Vec<String> = masters.split(',').map(|s| s.trim().to_string()).collect();
for master in &masters {
@@ -111,7 +121,10 @@ async fn main() -> Result<()> {
));
}
let fallback: FallbackOptions = args.storage_options.into_iter().collect();
let client = NamespaceClient::new(args.namespace.clone()).with_fallback(&fallback);
let signing_key = SigningKey::new(args.filer_signing_key);
let client = NamespaceClient::new(args.namespace.clone())
.with_fallback(&fallback)
.with_signing_key(signing_key.as_ref());
let mut service = VectorServer::new(client, fallback);
if args.read_cache_mb > 0 {
service = service.with_read_cache(ReadCache::new(args.read_cache_mb << 20));
@@ -125,7 +138,7 @@ async fn main() -> Result<()> {
.max_decoding_message_size(1 << 30)
.max_encoding_message_size(1 << 30),
)
.serve(args.listen.parse()?)
.serve_with_incoming(incoming)
.await?;
Ok(())
}
+3 -3
View File
@@ -424,13 +424,13 @@ impl VectorService for VectorServer {
if dimension == 0 || dimension > 4096 {
return Err(Status::invalid_argument("dimension must be 1..=4096"));
}
// The gateway has already created the table through the catalog.
let description = self
.client
.declare_table(&table.id, None)
.describe_table(&table.id)
.await
.map_err(internal)?;
let mut options = self.fallback.clone();
options.extend(description.storage_options.clone());
let options = dataset::table_options(&description, &self.fallback);
let schema = schema_for(dimension, metric, &req.non_filterable_metadata_keys);
let empty = RecordBatch::new_empty(schema.clone());
let reader = arrow_array::RecordBatchIterator::new(vec![Ok(empty)].into_iter(), schema);
+3 -3
View File
@@ -143,12 +143,12 @@ fn flip_horizontal_if(img: DynamicImage, do_flip: bool) -> DynamicImage {
DynamicImage::ImageRgba8(out)
}
/// Returns true if the given MIME type or file path extension indicates a JPEG file.
pub fn is_jpeg(mime_type: &str, path: &str) -> bool {
/// Returns true if the given MIME type or file name extension indicates a JPEG file.
pub fn is_jpeg(mime_type: &str, filename: &str) -> bool {
if mime_type == "image/jpeg" {
return true;
}
let lower = path.to_lowercase();
let lower = filename.to_lowercase();
lower.ends_with(".jpg") || lower.ends_with(".jpeg")
}
+412 -19
View File
@@ -5,21 +5,153 @@
use std::collections::HashMap;
use std::future::Future;
use std::pin::Pin;
use std::sync::{Arc, OnceLock, RwLock};
use std::task::{Context, Poll};
use std::time::Duration;
use aws_sdk_s3::Client;
use aws_sdk_s3::config::http::HttpResponse;
use aws_sdk_s3::config::timeout::TimeoutConfig;
use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
use aws_sdk_s3::error::{DisplayErrorContext, SdkError};
use aws_sdk_s3::operation::get_object::GetObjectError;
use aws_sdk_s3::operation::head_object::HeadObjectError;
use aws_sdk_s3::operation::upload_part::UploadPartOutput;
use aws_sdk_s3::operation::upload_part::builders::UploadPartFluentBuilder;
use aws_sdk_s3::primitives::{ByteStream, SdkBody};
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use bytes::Bytes;
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
use tokio::sync::Semaphore;
use tokio::sync::{Semaphore, watch};
/// Concurrency limit for multipart upload/download (matches Go's s3manager).
const CONCURRENCY: usize = 5;
/// A tier endpoint that drops packets or accepts connections without answering
/// must not hold a read, or a volume server's startup, for minutes per request.
/// Matches Go's s3ConnectTimeout and s3ReadTimeout. The SDK's stalled-stream
/// protection already fails a body that stops arriving.
const S3_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
const S3_READ_TIMEOUT: Duration = Duration::from_secs(60);
/// The SDK's read timeout runs from the start of the request, so on
/// upload_part it would also cap sending the part body. send_part bounds the
/// response wait instead, from when the body is written, like Go's
/// ResponseHeaderTimeout.
fn upload_part_config() -> aws_sdk_s3::config::Builder {
aws_sdk_s3::config::Builder::default().timeout_config(
TimeoutConfig::builder()
.connect_timeout(S3_CONNECT_TIMEOUT)
.disable_read_timeout()
.build(),
)
}
/// A part goes to the connection in chunks this size, so its last chunk is
/// taken only once nearly all of the part has been written.
const UPLOAD_CHUNK_SIZE: usize = 64 * 1024;
/// The latest attempt to send a part, and whether its body has been sent.
type PartSent = Arc<watch::Sender<(u64, bool)>>;
struct PartBody {
rest: Bytes,
attempt: u64,
sent: PartSent,
}
impl http_body::Body for PartBody {
type Data = Bytes;
type Error = std::convert::Infallible;
fn poll_frame(
mut self: Pin<&mut Self>,
_cx: &mut Context<'_>,
) -> Poll<Option<Result<http_body::Frame<Bytes>, Self::Error>>> {
if self.rest.is_empty() {
return Poll::Ready(None);
}
let n = self.rest.len().min(UPLOAD_CHUNK_SIZE);
let chunk = self.rest.split_to(n);
if self.rest.is_empty() {
let attempt = self.attempt;
self.sent.send_if_modified(|state| {
let latest = state.0 == attempt;
state.1 |= latest;
latest
});
}
Poll::Ready(Some(Ok(http_body::Frame::data(chunk))))
}
fn is_end_stream(&self) -> bool {
self.rest.is_empty()
}
fn size_hint(&self) -> http_body::SizeHint {
http_body::SizeHint::with_exact(self.rest.len() as u64)
}
}
/// Sends one multipart part, giving up once `response_timeout` passes after
/// the whole part was handed to the connection without a response. A retry
/// rebuilds the body and restarts that clock. The body keeps its in-memory
/// contents, so signing and the checksum header are unchanged.
async fn send_part(
request: UploadPartFluentBuilder,
data: Bytes,
response_timeout: Duration,
) -> Result<UploadPartOutput, String> {
let sent: PartSent = Arc::new(watch::channel((0, false)).0);
let body = {
let sent = sent.clone();
SdkBody::from(data.clone()).map_preserve_contents(move |_| {
let mut attempt = 0;
sent.send_modify(|state| {
*state = (state.0 + 1, data.is_empty());
attempt = state.0;
});
SdkBody::from_body_1_x(PartBody {
rest: data.clone(),
attempt,
sent: sent.clone(),
})
})
};
tokio::select! {
resp = request
.body(ByteStream::new(body))
.customize()
.config_override(upload_part_config())
.send() => resp.map_err(|e| DisplayErrorContext(&e).to_string()),
() = wait_unanswered(sent, response_timeout) => Err(format!(
"no response within {response_timeout:?} after sending the part"
)),
}
}
/// Completes once an attempt's body has been sent and `response_timeout`
/// passes before a later attempt starts.
async fn wait_unanswered(sent: PartSent, response_timeout: Duration) {
let mut state = sent.subscribe();
loop {
let attempt = state
.wait_for(|&(_, done)| done)
.await
.map_or(0, |state| state.0);
if tokio::time::timeout(
response_timeout,
state.wait_for(|&(latest, _)| latest != attempt),
)
.await
.is_err()
{
return;
}
}
}
/// A tier transfer failure. The variant is what callers match on; the
/// message is the operator-facing text.
#[derive(Debug, thiserror::Error)]
@@ -107,7 +239,13 @@ impl S3TierBackend {
.behavior_version(BehaviorVersion::latest())
.region(Region::new(region.to_string()))
.credentials_provider(credentials)
.force_path_style(config.force_path_style);
.force_path_style(config.force_path_style)
.timeout_config(
TimeoutConfig::builder()
.connect_timeout(S3_CONNECT_TIMEOUT)
.read_timeout(S3_READ_TIMEOUT)
.build(),
);
if !config.endpoint.is_empty() {
s3_config = s3_config.endpoint_url(&config.endpoint);
@@ -231,23 +369,23 @@ impl S3TierBackend {
TierError::Io(format!("failed to read file at offset {}: {}", off, e))
})?;
let upload_part_resp = client
.upload_part()
.bucket(&bucket)
.key(&key)
.upload_id(&upload_id)
.part_number(pn)
.body(buf.into())
.send()
.await
.map_err(|e| {
TierError::Io(format!(
"failed to upload part {} at offset {}: {}",
pn,
off,
DisplayErrorContext(&e)
))
})?;
let upload_part_resp = send_part(
client
.upload_part()
.bucket(&bucket)
.key(&key)
.upload_id(&upload_id)
.part_number(pn),
buf.into(),
S3_READ_TIMEOUT,
)
.await
.map_err(|e| {
TierError::Io(format!(
"failed to upload part {} at offset {}: {}",
pn, off, e
))
})?;
let e_tag = upload_part_resp.e_tag().unwrap_or_default().to_string();
@@ -560,6 +698,25 @@ impl S3TierBackend {
})
}
/// `read_range_blocking` that gives up once `timeout` passes, returning
/// `Ok(None)`.
pub fn read_range_blocking_within(
self: &Arc<Self>,
key: &str,
offset: u64,
size: usize,
timeout: Duration,
) -> Result<Option<Vec<u8>>, TierError> {
let backend = Arc::clone(self);
let key = key.to_string();
block_on_tier_future(async move {
match tokio::time::timeout(timeout, backend.read_range(&key, offset, size)).await {
Ok(read) => read.map(Some),
Err(_) => Ok(None),
}
})
}
pub fn read_range_blocking(
&self,
key: &str,
@@ -725,6 +882,13 @@ fn describe_join_error(join_error: tokio::task::JoinError) -> TierError {
mod tests {
use super::*;
use crate::remote_storage::s3::tests::{CannedResponse, NO_SUCH_KEY};
use aws_sdk_s3::config::http::HttpRequest;
use aws_sdk_s3::config::{HttpClient, RuntimeComponents};
use aws_smithy_runtime_api::client::http::{
HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector,
};
use aws_smithy_runtime_api::http::StatusCode;
use http_body::Body as _;
use std::collections::HashSet;
use tokio::runtime::Handle;
@@ -897,6 +1061,235 @@ mod tests {
assert!(matches!(err, TierError::Io(_)), "{err:?}");
}
#[test]
fn backend_bounds_connect_and_read_waits() {
let (_listener, backend) = silent_backend();
let timeouts = backend.client.config().timeout_config().unwrap();
assert_eq!(timeouts.connect_timeout(), Some(S3_CONNECT_TIMEOUT));
assert_eq!(timeouts.read_timeout(), Some(S3_READ_TIMEOUT));
}
#[derive(Debug)]
struct CaptureTimeouts(Arc<std::sync::Mutex<Option<TimeoutConfig>>>);
impl aws_sdk_s3::config::Intercept for CaptureTimeouts {
fn name(&self) -> &'static str {
"CaptureTimeouts"
}
fn read_before_transmit(
&self,
_context: &aws_sdk_s3::config::interceptors::BeforeTransmitInterceptorContextRef<'_>,
_runtime_components: &aws_sdk_s3::config::RuntimeComponents,
cfg: &mut aws_sdk_s3::config::ConfigBag,
) -> Result<(), aws_sdk_s3::error::BoxError> {
*self.0.lock().unwrap() = cfg.load::<TimeoutConfig>().cloned();
Err("captured".into())
}
}
#[tokio::test]
async fn upload_part_keeps_the_connect_timeout_without_the_read_timeout() {
let (_listener, backend) = silent_backend();
let seen = Arc::new(std::sync::Mutex::new(None));
let _ = backend
.client
.upload_part()
.bucket("bucket")
.key("key")
.upload_id("upload")
.part_number(1)
.body(vec![0u8; 8].into())
.customize()
.config_override(upload_part_config().interceptor(CaptureTimeouts(seen.clone())))
.send()
.await;
let timeouts = seen
.lock()
.unwrap()
.clone()
.expect("request never reached transmit");
assert_eq!(timeouts.connect_timeout(), Some(S3_CONNECT_TIMEOUT));
assert_eq!(timeouts.read_timeout(), None);
}
fn upload_part_request(backend: &S3TierBackend) -> UploadPartFluentBuilder {
backend
.client
.upload_part()
.bucket("bucket")
.key("key")
.upload_id("upload")
.part_number(1)
}
#[tokio::test]
async fn send_part_gives_up_when_no_response_follows_the_body() {
let (_listener, backend) = silent_backend();
let sent = tokio::time::timeout(
Duration::from_secs(30),
send_part(
upload_part_request(&backend),
Bytes::from_static(&[0u8; 8]),
Duration::from_millis(200),
),
)
.await
.expect("a part with no response did not give up");
let err = sent.unwrap_err();
assert!(err.starts_with("no response within 200ms"), "{err}");
}
#[tokio::test(start_paused = true)]
async fn a_retry_restarts_the_response_clock() {
let sent: PartSent = Arc::new(watch::channel((1, true)).0);
let mut unanswered =
std::pin::pin!(wait_unanswered(sent.clone(), Duration::from_millis(200)));
let wait = |ms| tokio::time::sleep(Duration::from_millis(ms));
tokio::select! {
() = unanswered.as_mut() => panic!("gave up early"),
() = wait(150) => {}
}
// The retry starts and finishes sending before the waiter runs again.
sent.send_replace((2, false));
sent.send_replace((2, true));
tokio::select! {
() = unanswered.as_mut() => panic!("kept the first attempt's clock"),
() = wait(150) => {}
}
tokio::select! {
() = unanswered => {}
() = wait(100) => panic!("the retry never timed out"),
}
}
/// Takes the request body one chunk per `pause`, then answers at once.
#[derive(Debug, Clone)]
struct SlowReader {
pause: Duration,
}
impl HttpConnector for SlowReader {
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
let pause = self.pause;
HttpConnectorFuture::new(async move {
let mut body = std::pin::pin!(request.into_body());
loop {
tokio::time::sleep(pause).await;
let frame = std::future::poll_fn(|cx| body.as_mut().poll_frame(cx)).await;
if frame.is_none() || body.is_end_stream() {
break;
}
}
let mut response =
HttpResponse::new(StatusCode::try_from(200).unwrap(), SdkBody::empty());
response.headers_mut().insert("ETag", "\"etag\"");
Ok(response)
})
}
}
impl HttpClient for SlowReader {
fn http_connector(
&self,
_settings: &HttpConnectorSettings,
_components: &RuntimeComponents,
) -> SharedHttpConnector {
SharedHttpConnector::new(self.clone())
}
}
#[tokio::test]
async fn send_part_does_not_cap_a_slow_body() {
let config = aws_sdk_s3::Config::builder()
.behavior_version(BehaviorVersion::latest())
.region(Region::new("us-east-1"))
.credentials_provider(Credentials::new("AKIATEST", "secret", None, None, "test"))
.endpoint_url("http://127.0.0.1:1")
.force_path_style(true)
.http_client(SlowReader {
pause: Duration::from_millis(300),
})
.build();
let backend = S3TierBackend {
client: Client::from_conf(config),
bucket: "bucket".to_string(),
storage_class: "STANDARD".to_string(),
};
let output = send_part(
upload_part_request(&backend),
Bytes::from(vec![0u8; 4 * UPLOAD_CHUNK_SIZE]),
Duration::from_millis(200),
)
.await
.expect("a part still being sent was cut short");
assert_eq!(output.e_tag(), Some("\"etag\""));
}
#[test]
fn read_range_blocking_within_gives_up_on_a_silent_endpoint() {
let (_listener, backend) = silent_backend();
let start = std::time::Instant::now();
let got = backend
.read_range_blocking_within("key", 0, 8, Duration::from_millis(200))
.unwrap();
assert!(got.is_none());
assert!(start.elapsed() < Duration::from_secs(5));
}
#[tokio::test]
async fn read_range_gives_up_when_the_body_stalls() {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let endpoint = format!("http://{}", listener.local_addr().unwrap());
std::thread::spawn(move || {
let mut held = Vec::new();
for mut conn in listener.incoming().flatten() {
let mut request = Vec::new();
let mut byte = [0u8; 1];
while !request.ends_with(b"\r\n\r\n") {
match std::io::Read::read(&mut conn, &mut byte) {
Ok(1) => request.push(byte[0]),
_ => break,
}
}
let _ = std::io::Write::write_all(
&mut conn,
b"HTTP/1.1 206 Partial Content\r\nContent-Length: 8\r\nContent-Range: bytes 0-7/1024\r\n\r\n",
);
held.push(conn);
}
});
let backend = S3TierBackend::new(&S3TierConfig {
access_key: "access".to_string(),
secret_key: "secret".to_string(),
region: "us-east-1".to_string(),
bucket: "bucket".to_string(),
endpoint,
storage_class: String::new(),
force_path_style: true,
});
let read = tokio::time::timeout(Duration::from_secs(30), backend.read_range("key", 0, 8))
.await
.expect("a read whose body stalls did not give up");
let err = read.unwrap_err();
assert!(matches!(err, TierError::Io(_)), "{err:?}");
}
fn silent_backend() -> (std::net::TcpListener, Arc<S3TierBackend>) {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let backend = S3TierBackend::new(&S3TierConfig {
access_key: "access".to_string(),
secret_key: "secret".to_string(),
region: "us-east-1".to_string(),
bucket: "bucket".to_string(),
endpoint: format!("http://{}", listener.local_addr().unwrap()),
storage_class: String::new(),
force_path_style: true,
});
(listener, Arc::new(backend))
}
#[test]
fn read_range_blocking_no_such_key_is_not_found() {
let err = backend_answering(404, NO_SUCH_KEY)
+422 -44
View File
@@ -733,6 +733,27 @@ pub(crate) async fn check_delete_replicas(
Ok(())
}
/// `query` with the fields named in `overrides` replaced by their values.
fn override_query_fields(query: &str, overrides: &[(&str, String)]) -> Result<String, String> {
let mut fields = Vec::new();
for field in query.split('&').filter(|field| !field.is_empty()) {
let decoded: Vec<(String, String)> = serde_urlencoded::from_str(field)
.map_err(|e| format!("parse replication query: {e}"))?;
if !decoded
.iter()
.any(|(name, _)| overrides.iter().any(|(o, _)| o == name))
{
fields.push(field);
}
}
let encoded = serde_urlencoded::to_string(overrides)
.map_err(|e| format!("encode replication query: {e}"))?;
if !encoded.is_empty() {
fields.push(&encoded);
}
Ok(fields.join("&"))
}
/// Helper to synchronously replicate a request to peer volume servers.
async fn do_replicated_request(
state: &VolumeServerState,
@@ -794,16 +815,7 @@ async fn do_replicated_request(
return Ok(());
}
let mut query_fields = Vec::new();
for field in query.split('&').filter(|field| !field.is_empty()) {
let decoded: Vec<(String, String)> = serde_urlencoded::from_str(field)
.map_err(|e| format!("parse replication query: {e}"))?;
if !decoded.iter().any(|(name, _)| name == "type") {
query_fields.push(field);
}
}
query_fields.push("type=replicate");
let new_query = query_fields.join("&");
let new_query = override_query_fields(query, &[("type", "replicate".to_string())])?;
let mut futures = Vec::new();
for loc in remote_locations {
@@ -831,6 +843,12 @@ async fn do_replicated_request(
if let Some(auth) = headers.get(axum::http::header::AUTHORIZATION) {
req_builder = req_builder.header(axum::http::header::AUTHORIZATION, auth);
}
for (name, value) in headers
.iter()
.filter(|(name, _)| name.as_str().starts_with("seaweed-"))
{
req_builder = req_builder.header(name, value);
}
if let Some(ref b) = body {
req_builder = req_builder.body(b.clone());
@@ -2893,8 +2911,12 @@ pub async fn post_handler(
while let Ok(Some(field)) = multipart.next_field().await {
let field_name = field.name().map(|s| s.to_string());
let fname = field.file_name().map(clean_windows_path_base);
let fct = field.content_type().map(|m| m.to_string());
let field_headers = field.headers().clone();
let fct = field_headers
.get(header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.filter(|s| !s.is_empty())
.map(|s| s.to_string());
let fce = field_headers
.get(header::CONTENT_ENCODING)
.and_then(|v| v.to_str().ok())
@@ -3060,15 +3082,18 @@ pub async fn post_handler(
let mime_type = if let Some(ref pct) = parsed_content_type {
pct.clone()
} else {
let multipart_fallback =
if should_parse_multipart && !filename.is_empty() && !is_chunk_manifest {
mime_guess::from_path(&filename)
.first()
.map(|m| m.to_string())
.unwrap_or_default()
} else {
String::new()
};
let multipart_fallback = if should_parse_multipart
&& !filename.is_empty()
&& !is_chunk_manifest
&& !is_replicate
{
mime_guess::from_path(&filename)
.first()
.map(|m| m.to_string())
.unwrap_or_default()
} else {
String::new()
};
headers
.get(header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
@@ -3131,23 +3156,24 @@ pub async fn post_handler(
.collect();
// Fix JPEG orientation from EXIF data before storing (matches Go behavior).
let body_data = if state.fix_jpg_orientation && crate::images::is_jpeg(&mime_type, &path) {
crate::images::fix_jpg_orientation(&body_data_raw)
} else {
body_data_raw
};
let (body_data, body_rewritten) =
if state.fix_jpg_orientation && crate::images::is_jpeg(&mime_type, &filename) {
let fixed = crate::images::fix_jpg_orientation(&body_data_raw);
let rewritten = fixed != body_data_raw;
(fixed, rewritten)
} else {
(body_data_raw, false)
};
// Auto-compress compressible file types (matches Go's IsCompressableFileType).
// Only compress if not already gzipped and compression saves >10%.
// Go uses filepath.Base(pu.FileName) for extension detection (not the URL path).
let (final_data, final_is_gzipped) = if !is_gzipped && !is_chunk_manifest {
let ext = {
let dot_pos = filename.rfind('.');
dot_pos
.map(|p| filename[p..].to_lowercase())
.unwrap_or_default()
};
if is_compressible_file_type(&ext, &mime_type) {
// A replica keeps the compression state the primary stored.
let (final_data, final_is_gzipped) = if !is_gzipped && !is_chunk_manifest && !is_replicate {
let ext = filename
.rfind('.')
.map(|p| &filename[p..])
.unwrap_or_default();
if is_compressible_file_type(ext, &mime_type) {
if let Some(compressed) = try_gzip_data(&body_data) {
if compressed.len() * 10 < body_data.len() * 9 {
(compressed, true)
@@ -3235,23 +3261,55 @@ pub async fn post_handler(
};
// The fan-out runs concurrently with the local needle write, like Go's
// ReplicatedWrite: it reads the already-buffered request body, so the two
// sides share no mutable state. The ack still waits for both.
// ReplicatedWrite: replicas get the stored bytes with their compression
// state, so they keep it instead of re-deriving it. The ack waits for both.
let replication = if needs_replication {
let state_clone = state.clone();
let path_clone = path.clone();
let query_clone = query.clone();
let headers_clone = headers.clone();
let body_clone = body.clone();
let mut overrides = vec![("ts", n.last_modified.to_string())];
if let Some(t) = ttl.filter(|t| !t.is_empty()) {
overrides.push(("ttl", t.to_string()));
}
if is_chunk_manifest {
overrides.push(("cm", "true".to_string()));
}
if fsync {
overrides.push(("fsync", "true".to_string()));
}
let mut replica_headers = HeaderMap::new();
if let Some(auth) = headers.get(header::AUTHORIZATION) {
replica_headers.insert(header::AUTHORIZATION, auth.clone());
}
if n.has_pairs() {
for (key, value) in &pair_map {
if let (Ok(name), Ok(value)) = (
header::HeaderName::try_from(format!("seaweed-{key}")),
header::HeaderValue::from_str(value),
) {
replica_headers.insert(name, value);
}
}
}
let (content_type, replica_body) = replica_upload_body(
&filename,
&mime_type,
n.is_compressed().then_some("gzip"),
original_content_md5.as_deref().filter(|_| !body_rewritten),
&n.data,
);
if let Ok(value) = header::HeaderValue::from_str(&content_type) {
replica_headers.insert(header::CONTENT_TYPE, value);
}
Some(tokio::spawn(async move {
do_replicated_request(
&state_clone,
vid.0,
Method::POST,
&path_clone,
&query_clone,
&headers_clone,
Some(body_clone),
&override_query_fields(&query_clone, &overrides)?,
&replica_headers,
Some(replica_body),
)
.await
}))
@@ -4766,6 +4824,15 @@ fn extract_jwt(headers: &HeaderMap, uri: &axum::http::Uri) -> Option<String> {
/// Check if a file type should be compressed based on extension and MIME type.
/// Returns true only when we are sure the type is compressible.
fn is_compressible_file_type(ext: &str, mtype: &str) -> bool {
let ext = ext.to_ascii_lowercase();
let ext = ext.as_str();
let mtype = mtype
.split(';')
.next()
.unwrap_or_default()
.trim()
.to_ascii_lowercase();
let mtype = mtype.as_str();
// text/*
if mtype.starts_with("text/") {
return true;
@@ -4798,6 +4865,9 @@ fn is_compressible_file_type(ext: &str, mtype: &str) -> bool {
if mtype.ends_with("script") {
return true;
}
if mtype.ends_with("json") || mtype.ends_with("json-seq") {
return true;
}
if mtype.ends_with("vnd.rar") {
return false;
}
@@ -4811,6 +4881,41 @@ fn is_compressible_file_type(ext: &str, mtype: &str) -> bool {
false
}
/// One-part multipart upload of a stored needle, the form Go's upload_content
/// sends: the bytes as stored, labeled with their content encoding.
fn replica_upload_body(
filename: &str,
mime: &str,
content_encoding: Option<&str>,
content_md5: Option<&str>,
data: &[u8],
) -> (String, bytes::Bytes) {
let boundary = uuid::Uuid::new_v4().simple().to_string();
let mut head = format!(
"--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{}\"\r\n",
filename.replace('\\', "\\\\").replace('"', "\\\"")
);
if !mime.is_empty() {
head.push_str(&format!("Content-Type: {mime}\r\n"));
}
if let Some(encoding) = content_encoding {
head.push_str(&format!("Content-Encoding: {encoding}\r\n"));
}
if let Some(md5) = content_md5 {
head.push_str(&format!("Content-MD5: {md5}\r\n"));
}
head.push_str("\r\n");
let tail = format!("\r\n--{boundary}--\r\n");
let mut body = Vec::with_capacity(head.len() + data.len() + tail.len());
body.extend_from_slice(head.as_bytes());
body.extend_from_slice(data);
body.extend_from_slice(tail.as_bytes());
(
format!("multipart/form-data; boundary={boundary}"),
body.into(),
)
}
/// Try to gzip data. Returns None on error.
fn try_gzip_data(data: &[u8]) -> Option<Vec<u8>> {
use flate2::Compression;
@@ -5283,6 +5388,18 @@ mod tests {
assert!(is_compressible_file_type("", "application/javascript"));
assert!(!is_compressible_file_type("", "application/zstd"));
assert!(!is_compressible_file_type("", "application/vnd.rar"));
assert!(is_compressible_file_type("", "application/json"));
assert!(is_compressible_file_type(
"",
"application/json; charset=utf-8"
));
assert!(is_compressible_file_type("", "Application/JSON"));
assert!(is_compressible_file_type("", "application/ld+json"));
assert!(is_compressible_file_type("", "application/x-ndjson"));
assert!(is_compressible_file_type("", "application/json-seq"));
assert!(is_compressible_file_type("", "text/plain; charset=utf-8"));
assert!(is_compressible_file_type(".JSON", ""));
assert!(!is_compressible_file_type(".gz", "application/json"));
// Audio
assert!(is_compressible_file_type(".wav", "audio/wav"));
@@ -8245,12 +8362,15 @@ mod tests {
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
*state.current_master_url.write().await = listener.local_addr().unwrap().to_string();
let master_server = tokio::spawn(async move { axum::serve(listener, master).await.unwrap() });
let master_server =
tokio::spawn(async move { axum::serve(listener, master).await.unwrap() });
let id = 0x6e7a_0c05u64;
let fid = format!("1,{id:x}{TEST_COOKIE:08x}");
let token = gen_jwt(&key, 60, &fid).unwrap();
let query = "%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&%74ype=client&type=old";
let expected_query = "%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&type=replicate";
let query =
"%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&ts=1700000000&%74ype=client&type=old";
let expected_query =
"%6Awt=expired&tag=a%2Fb+%2B&tag=x%20y&bare&ts=1700000000&type=replicate";
let payload = b"authenticated replica bytes";
for method in [Method::POST, Method::DELETE] {
let response = tokio::time::timeout(
@@ -8299,6 +8419,264 @@ mod tests {
peer_server.abort();
}
#[tokio::test]
async fn http_replica_stores_the_primary_needle() {
use crate::storage::super_block::ReplicaPlacement;
use axum::{Json, Router, routing::get};
use tower::ServiceExt;
let tmp = tempfile::TempDir::new().unwrap();
let mut state = volume_test_state(&tmp);
Arc::get_mut(&mut state).unwrap().self_url = "127.0.0.1:8080".to_string();
let volume = state
.store
.read()
.unwrap()
.find_volume(VolumeId(1))
.unwrap()
.1;
volume
.set_replica_placement(ReplicaPlacement::from_string("001").unwrap())
.unwrap();
let peer_tmp = tempfile::TempDir::new().unwrap();
let peer = volume_test_state(&peer_tmp);
let peer_volume = peer
.store
.read()
.unwrap()
.find_volume(VolumeId(1))
.unwrap()
.1;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let peer_addr = listener.local_addr().unwrap();
let peer_server = tokio::spawn(async move {
axum::serve(
listener,
super::super::volume_server::build_admin_router(peer),
)
.await
.unwrap();
});
let master = Router::new().route(
"/dir/lookup",
get(move || async move {
Json(serde_json::json!({"locations": [
{"url": "127.0.0.1:8080"},
{"url": peer_addr.to_string()}
]}))
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
*state.current_master_url.write().await = listener.local_addr().unwrap().to_string();
let master_server =
tokio::spawn(async move { axum::serve(listener, master).await.unwrap() });
let payload = "compressible replica text ".repeat(64);
let boundary = "replica-boundary";
let multipart = format!(
"--{boundary}\r\nContent-Disposition: form-data; name=\"ttl\"\r\n\r\n3m\r\n\
--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"a \\\"b\\\".txt\"\r\n\
Content-Type: text/plain\r\n\r\n{payload}\r\n--{boundary}--\r\n"
);
let cases = [
(
0x6e7a_0c06u64,
Request::builder()
.header(header::CONTENT_TYPE, "text/plain")
.header(
header::CONTENT_DISPOSITION,
"attachment; filename=\"notes.txt\"",
)
.header("Content-MD5", compute_md5_base64(payload.as_bytes()))
.header("Seaweed-Color", "blue"),
Body::from(payload.clone()),
),
(
0x6e7a_0c07u64,
Request::builder().header(
header::CONTENT_TYPE,
format!("multipart/form-data; boundary={boundary}"),
),
Body::from(multipart),
),
(
0x6e7a_0c0au64,
Request::builder().header(
header::CONTENT_DISPOSITION,
"attachment; filename=\"untyped.txt\"",
),
Body::from(payload.clone()),
),
(
0x6e7a_0c0bu64,
Request::builder().header(
header::CONTENT_DISPOSITION,
format!("attachment; filename=\"{}.txt\"", "n".repeat(300)),
),
Body::from(payload.clone()),
),
(
0x6e7a_0c0eu64,
Request::builder()
.header(header::CONTENT_TYPE, "Text/Plain; charset=UTF-8")
.header(
header::CONTENT_DISPOSITION,
"attachment; filename=\"upper.txt\"",
),
Body::from(payload.clone()),
),
];
for (id, request, body) in cases {
let response = tokio::time::timeout(
std::time::Duration::from_secs(5),
super::super::volume_server::build_admin_router(state.clone()).oneshot(
request
.method(Method::POST)
.uri(format!("/1,{id:x}{TEST_COOKIE:08x}?ts=1700000000"))
.body(body)
.unwrap(),
),
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
let read = |v: &Arc<crate::storage::volume::Volume>| {
let mut stored = Needle {
id: NeedleId(id),
cookie: Cookie(TEST_COOKIE),
..Needle::default()
};
v.read_needle(&mut stored).unwrap();
stored
};
let (primary, replica) = (read(&volume), read(&peer_volume));
assert!(primary.is_compressed());
assert_eq!(
maybe_decompress_gzip(&primary.data).unwrap(),
payload.as_bytes()
);
assert_eq!(replica.data, primary.data);
assert_eq!(replica.flags, primary.flags);
assert_eq!(replica.name, primary.name);
assert_eq!(replica.mime, primary.mime);
assert_eq!(replica.pairs, primary.pairs);
assert_eq!(replica.ttl, primary.ttl);
assert_eq!(replica.last_modified, 1700000000);
}
master_server.abort();
peer_server.abort();
}
#[tokio::test]
async fn http_replica_keeps_uncompressed_bytes() {
use tower::ServiceExt;
let tmp = tempfile::TempDir::new().unwrap();
let state = volume_test_state(&tmp);
let volume = state
.store
.read()
.unwrap()
.find_volume(VolumeId(1))
.unwrap()
.1;
let payload = "compressible replica text ".repeat(64);
for (id, query, compressed) in [
(0x6e7a_0c08u64, "?type=replicate", false),
(0x6e7a_0c09u64, "", true),
] {
let response = super::super::volume_server::build_admin_router(state.clone())
.oneshot(
Request::builder()
.method(Method::POST)
.uri(format!("/1,{id:x}{TEST_COOKIE:08x}{query}"))
.header(header::CONTENT_TYPE, "text/plain")
.body(Body::from(payload.clone()))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
let mut stored = Needle {
id: NeedleId(id),
cookie: Cookie(TEST_COOKIE),
..Needle::default()
};
volume.read_needle(&mut stored).unwrap();
assert_eq!(stored.is_compressed(), compressed, "query {query:?}");
}
}
#[tokio::test]
async fn http_orientation_fix_follows_the_filename() {
use tower::ServiceExt;
let tmp = tempfile::TempDir::new().unwrap();
let mut state = volume_test_state(&tmp);
Arc::get_mut(&mut state).unwrap().fix_jpg_orientation = true;
let volume = state
.store
.read()
.unwrap()
.find_volume(VolumeId(1))
.unwrap()
.1;
let mut encoded = std::io::Cursor::new(Vec::new());
image::DynamicImage::ImageRgba8(image::RgbaImage::new(3, 2))
.write_to(&mut encoded, image::ImageFormat::Jpeg)
.unwrap();
let exif: &[u8] =
b"Exif\0\0MM\0\x2a\0\0\0\x08\0\x01\x01\x12\0\x03\0\0\0\x01\0\x03\0\0\0\0\0\0";
let mut tagged = vec![0xFF, 0xD8, 0xFF, 0xE1, 0, exif.len() as u8 + 2];
tagged.extend_from_slice(exif);
tagged.extend_from_slice(&encoded.into_inner()[2..]);
let upload = |id: u64, suffix: &'static str, filename: &'static str, body: Vec<u8>| {
let (state, volume) = (state.clone(), volume.clone());
async move {
let response = super::super::volume_server::build_admin_router(state)
.oneshot(
Request::builder()
.method(Method::POST)
.uri(format!("/1,{id:x}{TEST_COOKIE:08x}{suffix}"))
.header(header::CONTENT_TYPE, "application/octet-stream")
.header(
header::CONTENT_DISPOSITION,
format!("attachment; filename=\"{filename}\""),
)
.body(Body::from(body))
.unwrap(),
)
.await
.unwrap();
assert_eq!(
response.status(),
StatusCode::CREATED,
"{suffix} {filename}"
);
let mut stored = Needle {
id: NeedleId(id),
cookie: Cookie(TEST_COOKIE),
..Needle::default()
};
volume.read_needle(&mut stored).unwrap();
stored.data
}
};
let rotated = upload(0x6e7a_0c10, "", "photo.jpg", tagged.clone()).await;
assert_ne!(rotated, tagged);
assert_eq!(
upload(0x6e7a_0c11, ".jpg", "photo.png", tagged.clone()).await,
tagged
);
assert_eq!(
upload(0x6e7a_0c12, "?type=replicate", "photo.jpg", tagged.clone()).await,
rotated
);
assert_eq!(
upload(0x6e7a_0c13, "?type=replicate", "photo.jpg", rotated.clone()).await,
rotated
);
}
#[tokio::test]
async fn http_multipart_replica_refusal_is_not_marked() {
use tower::ServiceExt;
+376 -28
View File
@@ -9,14 +9,14 @@
//! Matches Go's storage/volume.go, volume_loading.go, volume_read.go,
//! volume_write.go, volume_super_block.go.
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use std::fs::{self, File, OpenOptions};
use std::io::{self, Read, Seek, SeekFrom, Write};
use std::ops::ControlFlow;
use std::path::Path;
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU8, AtomicU64, Ordering};
use std::sync::{Arc, Condvar, Mutex, RwLock};
use std::time::{SystemTime, UNIX_EPOCH};
use std::sync::{Arc, Condvar, LazyLock, Mutex, RwLock};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use tracing::{debug, error, info, warn};
@@ -1198,6 +1198,33 @@ impl RemoteDatFile {
.backend
.read_range_blocking(&self.key, offset, buf.len())
.map_err(io::Error::other)?;
Self::fill(buf, offset, data)
}
/// `read_exact_at` that fails with `TimedOut` once `timeout` passes.
pub(crate) fn read_exact_at_within(
&self,
buf: &mut [u8],
offset: u64,
timeout: Duration,
) -> io::Result<()> {
match self
.backend
.read_range_blocking_within(&self.key, offset, buf.len(), timeout)
.map_err(io::Error::other)?
{
Some(data) => Self::fill(buf, offset, data),
None => Err(io::Error::new(
io::ErrorKind::TimedOut,
format!(
"remote read at offset {} got no answer within {:?}",
offset, timeout
),
)),
}
}
fn fill(buf: &mut [u8], offset: u64, data: Vec<u8>) -> io::Result<()> {
if data.len() != buf.len() {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
@@ -1214,6 +1241,71 @@ impl RemoteDatFile {
}
}
/// A tiered volume loads without its superblock when the backend cannot serve
/// it; once a read stalls, the backend's other volumes skip it for a while.
/// Matches Go's remoteSuperBlockTimeout and remoteSuperBlockStallPeriod.
const REMOTE_SUPER_BLOCK_TIMEOUT: Duration = Duration::from_secs(10);
const REMOTE_SUPER_BLOCK_STALL_PERIOD: Duration = Duration::from_secs(60);
static REMOTE_SUPER_BLOCK_STALLS: LazyLock<Mutex<HashMap<String, Instant>>> =
LazyLock::new(Default::default);
fn read_super_block_bytes(
mut read: impl FnMut(&mut [u8], u64) -> Result<(), VolumeError>,
) -> Result<Vec<u8>, VolumeError> {
let mut header = [0u8; SUPER_BLOCK_SIZE];
read(&mut header, 0)?;
let extra_size = u16::from_be_bytes([header[6], header[7]]);
let mut full_buf = vec![0u8; SUPER_BLOCK_SIZE + extra_size as usize];
full_buf[..SUPER_BLOCK_SIZE].copy_from_slice(&header);
if extra_size > 0 {
read(&mut full_buf[SUPER_BLOCK_SIZE..], SUPER_BLOCK_SIZE as u64)?;
}
Ok(full_buf)
}
fn read_remote_super_block(
storage_name: &str,
remote: &RemoteDatFile,
timeout: Duration,
stall_period: Duration,
) -> Result<Vec<u8>, VolumeError> {
if let Some(until) = REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().get(storage_name)
&& Instant::now() < *until
{
return Err(VolumeError::Io(io::Error::other(format!(
"backend {} stalled on a super block read, skipped",
storage_name
))));
}
let deadline = Instant::now() + timeout;
let result = read_super_block_bytes(|buf, offset| {
remote
.read_exact_at_within(
buf,
offset,
deadline.saturating_duration_since(Instant::now()),
)
.map_err(VolumeError::from)
});
match &result {
Ok(_) => {
REMOTE_SUPER_BLOCK_STALLS
.lock()
.unwrap()
.remove(storage_name);
}
Err(VolumeError::Io(e)) if e.kind() == io::ErrorKind::TimedOut => {
REMOTE_SUPER_BLOCK_STALLS
.lock()
.unwrap()
.insert(storage_name.to_string(), Instant::now() + stall_period);
}
Err(_) => {}
}
result
}
// ============================================================================
// Volume
// ============================================================================
@@ -1684,6 +1776,7 @@ impl Volume {
)));
}
let mut remote_super_block_failed = false;
if already_has_super_block {
match self.read_super_block(io) {
Ok(()) => {
@@ -1700,6 +1793,12 @@ impl Volume {
error = %e,
"failed to read remote super block during load"
);
remote_super_block_failed = true;
let replication = self.volume_info.read().unwrap().replication.clone();
if !replication.is_empty() {
self.super_block.write().unwrap().replica_placement =
ReplicaPlacement::from_string(&replication)?;
}
}
Err(e) => return Err(e),
}
@@ -1748,17 +1847,24 @@ impl Volume {
// at the front of .idx. Best effort: a volume that cannot be
// repaired is still servable for everything the surviving rows
// index.
match self.repair_idx_head_tombstones(io) {
Ok(0) => {}
Ok(restored) => info!(
if remote_super_block_failed {
debug!(
volume_id = self.id.0,
restored, "recovered overwritten .idx rows from .dat"
),
Err(e) => warn!(
volume_id = self.id.0,
error = %e,
"recover overwritten .idx rows"
),
"remote backend unavailable, not recovering overwritten .idx rows on this load"
);
} else {
match self.repair_idx_head_tombstones(io) {
Ok(0) => {}
Ok(restored) => info!(
volume_id = self.id.0,
restored, "recovered overwritten .idx rows from .dat"
),
Err(e) => warn!(
volume_id = self.id.0,
error = %e,
"recover overwritten .idx rows"
),
}
}
// The permission bits still allow writing on a read-only mount;
@@ -2220,21 +2326,15 @@ impl Volume {
// ---- SuperBlock I/O ----
fn read_super_block(&self, io: &VolumeIo) -> Result<(), VolumeError> {
let mut header = [0u8; SUPER_BLOCK_SIZE];
self.read_exact_at_backend(io, &mut header, 0)?;
let extra_size = u16::from_be_bytes([header[6], header[7]]);
let total_size = SUPER_BLOCK_SIZE + extra_size as usize;
let mut full_buf = vec![0u8; total_size];
full_buf[..SUPER_BLOCK_SIZE].copy_from_slice(&header);
if extra_size > 0 {
self.read_exact_at_backend(
io,
&mut full_buf[SUPER_BLOCK_SIZE..],
SUPER_BLOCK_SIZE as u64,
)?;
}
let full_buf = match io.remote_dat_file.as_ref() {
Some(remote) if self.has_remote_file() => read_remote_super_block(
&self.remote_storage_name_key().0,
remote,
REMOTE_SUPER_BLOCK_TIMEOUT,
REMOTE_SUPER_BLOCK_STALL_PERIOD,
)?,
_ => read_super_block_bytes(|buf, offset| self.read_exact_at_backend(io, buf, offset))?,
};
*self.super_block.write().unwrap() = SuperBlock::from_bytes(&full_buf)?;
@@ -11523,6 +11623,254 @@ mod tests {
assert!(result.is_err(), "unopenable .dat must fail load");
}
fn silent_tier_backend() -> (
std::net::TcpListener,
Arc<crate::remote_storage::s3_tier::S3TierBackend>,
) {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let backend = crate::remote_storage::s3_tier::S3TierBackend::new(
&crate::remote_storage::s3_tier::S3TierConfig {
access_key: "access".to_string(),
secret_key: "secret".to_string(),
region: "us-east-1".to_string(),
bucket: "bucket-a".to_string(),
endpoint: format!("http://{}", listener.local_addr().unwrap()),
storage_class: "STANDARD".to_string(),
force_path_style: true,
},
);
(listener, Arc::new(backend))
}
#[test]
fn remote_super_block_read_gives_up_on_a_silent_backend_and_skips_it() {
let name = "s3.super_block_stall_test";
let (_listener, backend) = silent_tier_backend();
let silent = RemoteDatFile {
backend,
key: "remote-key".to_string(),
file_size: 1024,
modified_time: 0,
};
let start = Instant::now();
let err = read_remote_super_block(
name,
&silent,
Duration::from_millis(200),
Duration::from_secs(3600),
)
.unwrap_err();
assert!(
matches!(&err, VolumeError::Io(e) if e.kind() == io::ErrorKind::TimedOut),
"{err:?}"
);
assert!(start.elapsed() < Duration::from_secs(5));
let start = Instant::now();
assert!(
read_remote_super_block(
name,
&silent,
Duration::from_secs(30),
Duration::from_secs(3600)
)
.is_err()
);
assert!(
start.elapsed() < Duration::from_secs(1),
"a stalled backend must not be asked again within the stall period"
);
let sb = SuperBlock {
version: Version::current(),
..SuperBlock::default()
};
let (endpoint, shutdown_tx) = spawn_fake_s3_server(sb.to_bytes());
let answering = RemoteDatFile {
backend: Arc::new(crate::remote_storage::s3_tier::S3TierBackend::new(
&crate::remote_storage::s3_tier::S3TierConfig {
access_key: "access".to_string(),
secret_key: "secret".to_string(),
region: "us-east-1".to_string(),
bucket: "bucket-a".to_string(),
endpoint,
storage_class: "STANDARD".to_string(),
force_path_style: true,
},
)),
key: "remote-key".to_string(),
file_size: SUPER_BLOCK_SIZE as u64,
modified_time: 0,
};
REMOTE_SUPER_BLOCK_STALLS
.lock()
.unwrap()
.insert(name.to_string(), Instant::now());
let bytes = read_remote_super_block(
name,
&answering,
Duration::from_secs(10),
Duration::from_secs(3600),
)
.unwrap();
assert_eq!(
SuperBlock::from_bytes(&bytes).unwrap().version,
Version::current()
);
assert!(
!REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().contains_key(name),
"a successful read clears the stall"
);
let _ = shutdown_tx.send(());
}
#[test]
fn remote_volume_loads_without_waiting_on_a_stalled_backend() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let dat_len = {
let v = make_test_volume(dir);
v.dat_file_size().unwrap()
};
std::fs::remove_file(format!("{}/1.dat", dir)).unwrap();
let (_listener, backend) = silent_tier_backend();
let name = "s3.load_stall_test";
crate::remote_storage::s3_tier::global_s3_tier_registry()
.write()
.unwrap()
.register(name.to_string(), Arc::try_unwrap(backend).ok().unwrap());
REMOTE_SUPER_BLOCK_STALLS
.lock()
.unwrap()
.insert(name.to_string(), Instant::now() + Duration::from_secs(3600));
let vif = VifVolumeInfo {
files: vec![VifRemoteFile {
backend_type: "s3".to_string(),
backend_id: "load_stall_test".to_string(),
key: "remote-key".to_string(),
offset: 0,
file_size: dat_len,
modified_time: 123,
extension: ".dat".to_string(),
}],
version: Version::current().0 as u32,
replication: "001".to_string(),
..VifVolumeInfo::default()
};
std::fs::write(
format!("{}/1.vif", dir),
serde_json::to_string_pretty(&vif).unwrap(),
)
.unwrap();
let start = Instant::now();
let v = Volume::new(
dir,
dir,
VolumeId(1),
NeedleMapKind::InMemory,
&VolumeSpec::default(),
)
.unwrap();
assert!(start.elapsed() < Duration::from_secs(5));
assert!(v.has_remote_file());
assert_eq!(v.version(), Version::current());
assert_eq!(
v.super_block.read().unwrap().replica_placement,
ReplicaPlacement::from_string("001").unwrap()
);
crate::remote_storage::s3_tier::global_s3_tier_registry()
.write()
.unwrap()
.remove(name);
REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().remove(name);
}
#[test]
fn remote_volume_load_skips_idx_repair_on_a_stalled_backend() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let dat_len = {
let v = make_test_volume(dir);
for i in 1..=3u64 {
let mut n = Needle {
id: NeedleId(i),
cookie: Cookie(0x1234),
data: format!("needle-{i}").into_bytes(),
data_size: format!("needle-{i}").len() as u32,
..Needle::default()
};
v.write_needle(&mut n, true, false).unwrap();
}
v.dat_file_size().unwrap()
};
std::fs::remove_file(format!("{}/1.dat", dir)).unwrap();
let idx_path = format!("{}/1.idx", dir);
crate::storage::volume_idx_repair::tests::clobber_idx_head(&idx_path, &[4, 5]);
let idx_before = std::fs::read(&idx_path).unwrap();
let (_listener, backend) = silent_tier_backend();
let name = "s3.idx_repair_stall_test";
crate::remote_storage::s3_tier::global_s3_tier_registry()
.write()
.unwrap()
.register(name.to_string(), Arc::try_unwrap(backend).ok().unwrap());
REMOTE_SUPER_BLOCK_STALLS
.lock()
.unwrap()
.insert(name.to_string(), Instant::now() + Duration::from_secs(3600));
let vif = VifVolumeInfo {
files: vec![VifRemoteFile {
backend_type: "s3".to_string(),
backend_id: "idx_repair_stall_test".to_string(),
key: "remote-key".to_string(),
offset: 0,
file_size: dat_len,
modified_time: 123,
extension: ".dat".to_string(),
}],
version: Version::current().0 as u32,
..VifVolumeInfo::default()
};
std::fs::write(
format!("{}/1.vif", dir),
serde_json::to_string_pretty(&vif).unwrap(),
)
.unwrap();
let start = Instant::now();
let v = Volume::new(
dir,
dir,
VolumeId(1),
NeedleMapKind::InMemory,
&VolumeSpec::default(),
)
.unwrap();
assert!(
start.elapsed() < Duration::from_secs(5),
"the load waited on the stalled backend"
);
assert!(v.has_remote_file());
drop(v);
assert_eq!(
std::fs::read(&idx_path).unwrap(),
idx_before,
"the repair ran against an unreachable backend"
);
crate::remote_storage::s3_tier::global_s3_tier_registry()
.write()
.unwrap()
.remove(name);
REMOTE_SUPER_BLOCK_STALLS.lock().unwrap().remove(name);
}
#[test]
fn test_remote_only_volume_load_reads_from_tier_backend() {
let tmp = TempDir::new().unwrap();
@@ -224,7 +224,7 @@ fn finish_walk(walk: io::Result<()>) -> Result<(), VolumeError> {
}
#[cfg(test)]
mod tests {
pub(crate) mod tests {
use super::*;
use crate::storage::needle::crc::CRC;
use crate::storage::needle_map::NeedleMapKind;
@@ -267,7 +267,7 @@ mod tests {
/// Reproduce the damage a delete on a tiered read-only volume used to do: it
/// writes (key, offset 0, tombstone) rows over the front of .idx instead of
/// appending them.
fn clobber_idx_head(idx_path: &str, keys: &[u64]) {
pub(crate) fn clobber_idx_head(idx_path: &str, keys: &[u64]) {
let mut file = OpenOptions::new().write(true).open(idx_path).unwrap();
for (i, key) in keys.iter().enumerate() {
let mut row = Vec::new();
+5
View File
@@ -95,6 +95,11 @@ gateway without STS configured vends no credentials at all, so `--access-key`
and `--secret-key` supply a fallback; anything the namespace does vend wins over
them.
In a vector bucket's `vector` namespace the gateway vends write credentials only
to a caller that proves it holds `jwt.filer_signing.key`. Pass that key with
`--filer-signing-key` so compaction, cleanup and index builds can rewrite
vector indexes.
## State
All three jobs are implemented and tested end to end against a live gateway:
+18 -1
View File
@@ -33,9 +33,26 @@ pub fn admin_token(signing_key: &[u8], expires_after_secs: u64) -> Option<String
))
}
/// Derives the key for one use of a signing key, so a token signed with it is
/// not accepted where the signing key itself is.
pub fn derive_key(signing_key: &[u8], purpose: &str) -> Vec<u8> {
let key = hmac::Key::new(hmac::HMAC_SHA256, signing_key);
hmac::sign(&key, purpose.as_bytes()).as_ref().to_vec()
}
#[cfg(test)]
mod tests {
use super::admin_token;
use super::{admin_token, derive_key};
#[test]
fn derive_key_matches_the_go_derivation() {
let derived = derive_key(b"cluster-key", "seaweedfs-lance-cluster-token");
let hex: String = derived.iter().map(|b| format!("{b:02x}")).collect();
assert_eq!(
hex,
"b38c0b66da41621f2a9b6de68e4239ddd0d2dd7b4ede01aa91bd34a17edc1021"
);
}
#[test]
fn no_key_no_token() {
@@ -7,4 +7,4 @@
pub mod namespace;
pub use namespace::{NamespaceClient, TableDescription, parse_id};
pub use namespace::{NamespaceClient, SigningKey, TableDescription, parse_id};
@@ -8,6 +8,39 @@ use serde::Deserialize;
/// The delimiter the Lance namespace joins identifier parts with.
const DELIMITER: &str = "$";
/// Carries a token that proves the caller holds the cluster key, which marks
/// it as one of the cluster's own components.
const CLUSTER_TOKEN_HEADER: &str = "X-Seaweedfs-Cluster-Token";
/// A cluster token is minted for each request, so it only has to outlast the
/// clock skew between this host and the gateway.
const CLUSTER_TOKEN_SECS: u64 = 300;
/// The key cluster tokens are signed with, derived from the cluster key
/// (`jwt.filer_signing.key`) the way the gateway derives it, so a token
/// captured on the wire does not authorize admin calls. The catalog vends
/// credentials that write a vector bucket's index data only to callers that
/// send one. Not Debug, so the key cannot reach a log.
#[derive(Clone)]
pub struct SigningKey {
key: Vec<u8>,
}
impl SigningKey {
pub fn new(key: Option<String>) -> Option<Self> {
key.filter(|key| !key.is_empty()).map(|key| Self {
key: seaweed_worker_core::token::derive_key(
key.as_bytes(),
"seaweedfs-lance-cluster-token",
),
})
}
fn token(&self) -> Option<String> {
seaweed_worker_core::token::admin_token(&self.key, CLUSTER_TOKEN_SECS)
}
}
#[derive(Debug, Deserialize)]
pub struct TableDescription {
pub location: String,
@@ -35,6 +68,7 @@ pub struct NamespaceClient {
base_url: String,
http: reqwest::Client,
api_key: Option<String>,
signing_key: Option<SigningKey>,
}
/// A namespace call that has not answered by now is not going to. Without this
@@ -71,6 +105,7 @@ impl NamespaceClient {
base_url: base_url.into().trim_end_matches('/').to_string(),
http,
api_key: None,
signing_key: None,
}
}
@@ -92,10 +127,25 @@ impl NamespaceClient {
self
}
pub fn with_signing_key(mut self, signing_key: Option<&SigningKey>) -> Self {
if signing_key.is_some() && !protected_transport(&self.base_url) {
tracing::warn!(
"catalog url {} exposes the cluster token in cleartext",
self.base_url
);
}
self.signing_key = signing_key.cloned();
self
}
fn authed(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
match &self.api_key {
let request = match &self.api_key {
Some(key) => request.header("x-api-key", key),
None => request,
};
match self.signing_key.as_ref().and_then(SigningKey::token) {
Some(token) => request.header(CLUSTER_TOKEN_HEADER, token),
None => request,
}
}
@@ -224,7 +274,74 @@ pub fn parse_id(encoded: &str) -> Vec<String> {
#[cfg(test)]
mod tests {
use super::{MAX_REDIRECT_HOPS, may_follow, protected_transport, same_origin};
use super::{
CLUSTER_TOKEN_SECS, MAX_REDIRECT_HOPS, NamespaceClient, SigningKey, may_follow,
protected_transport, same_origin,
};
/// A catalog that answers one request with a table location and hands back
/// the request it read.
async fn recording_catalog() -> (String, tokio::sync::oneshot::Receiver<String>) {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let (sender, receiver) = tokio::sync::oneshot::channel();
tokio::spawn(async move {
let (mut conn, _) = listener.accept().await.unwrap();
let mut request = vec![0u8; 8192];
let n = conn.read(&mut request).await.unwrap();
let _ = sender.send(String::from_utf8_lossy(&request[..n]).into_owned());
let body = r#"{"location":"s3://b/vector/t"}"#;
let response = format!(
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
body.len()
);
let _ = conn.write_all(response.as_bytes()).await;
});
(url, receiver)
}
fn header<'a>(request: &'a str, name: &str) -> Option<&'a str> {
request.lines().find_map(|line| {
let (key, value) = line.split_once(':')?;
key.eq_ignore_ascii_case(name).then(|| value.trim())
})
}
#[tokio::test]
async fn signing_key_sends_a_cluster_token() {
let id = ["b".to_string(), "vector".to_string(), "t".to_string()];
let (url, request) = recording_catalog().await;
let key = SigningKey::new(Some("cluster-key".into()));
NamespaceClient::new(url)
.with_signing_key(key.as_ref())
.describe_table(&id)
.await
.unwrap();
let request = request.await.unwrap();
let token = header(&request, "x-seaweedfs-cluster-token").expect("cluster token");
let raw =
seaweed_worker_core::token::admin_token(b"cluster-key", CLUSTER_TOKEN_SECS).unwrap();
assert_eq!(token.split('.').count(), 3);
assert_ne!(
token.rsplit('.').next(),
raw.rsplit('.').next(),
"signed with the cluster key itself"
);
let (url, request) = recording_catalog().await;
NamespaceClient::new(url).describe_table(&id).await.unwrap();
assert_eq!(
header(&request.await.unwrap(), "x-seaweedfs-cluster-token"),
None
);
}
#[test]
fn an_empty_signing_key_is_no_key() {
assert!(SigningKey::new(None).is_none());
assert!(SigningKey::new(Some(String::new())).is_none());
}
#[test]
fn may_follow_bounds_hops_within_the_origin() {
+59 -11
View File
@@ -65,21 +65,38 @@ pub async fn open_with(
})
}
/// The namespace vends object_store's own option names, so they pass straight
/// through. What it vends always wins: the fallback exists because a deployment
/// without STS vends no credentials at all, and then the worker has no other way
/// to reach the data.
/// The object_store options for a table. The namespace vends object_store's own
/// option names, so they pass straight through. What it vends always wins: the
/// fallback exists because a deployment without STS vends no credentials at
/// all, and then the worker has no other way to reach the data.
pub fn table_options(
description: &TableDescription,
fallback: &FallbackOptions,
) -> HashMap<String, String> {
let mut options: HashMap<String, String> = fallback.clone();
options.extend(description.storage_options.clone());
// A store that takes unsigned requests needs no credentials, but without
// any lance builds the AWS default chain, and that probes instance
// metadata on every open.
if options
.get("aws_skip_signature")
.is_some_and(|v| v == "true")
{
options.entry("aws_access_key_id".into()).or_default();
options.entry("aws_secret_access_key".into()).or_default();
}
options
}
async fn open_at(
description: &TableDescription,
fallback: &FallbackOptions,
wrapper: Option<Arc<dyn WrappingObjectStore>>,
) -> Result<Dataset> {
let mut options: HashMap<String, String> = fallback.clone();
options.extend(description.storage_options.clone());
DatasetBuilder::from_uri(&description.location)
.with_store_params(ObjectStoreParams {
storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
options,
table_options(description, fallback),
))),
object_store_wrapper: wrapper,
..Default::default()
@@ -102,14 +119,12 @@ pub async fn drop(
Err(err) => return Err(err),
};
check_drop_location(id, &description.location)?;
let mut options: HashMap<String, String> = fallback.clone();
options.extend(description.storage_options.clone());
let (store, path) = ObjectStore::from_uri_and_params(
Arc::new(ObjectStoreRegistry::default()),
&description.location,
&ObjectStoreParams {
storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
options,
table_options(&description, fallback),
))),
..Default::default()
},
@@ -203,12 +218,45 @@ fn schema_json(dataset: &Dataset) -> Option<String> {
#[cfg(test)]
mod tests {
use super::{check_drop_location, table_location};
use super::{FallbackOptions, check_drop_location, table_location, table_options};
use crate::catalog::TableDescription;
fn id(parts: &[&str]) -> Vec<String> {
parts.iter().map(|part| part.to_string()).collect()
}
fn options(pairs: &[(&str, &str)]) -> FallbackOptions {
pairs
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect()
}
#[test]
fn table_options_keep_an_unsigned_store_off_the_aws_default_chain() {
let described = |vended: &[(&str, &str)]| TableDescription {
location: "s3://bkt/vector/x".into(),
storage_options: options(vended),
managed_versioning: false,
};
let unsigned = described(&[("aws_skip_signature", "true")]);
let got = table_options(&unsigned, &FallbackOptions::new());
assert_eq!(got.get("aws_access_key_id").map(String::as_str), Some(""));
assert_eq!(
got.get("aws_secret_access_key").map(String::as_str),
Some("")
);
let fallback = options(&[("aws_access_key_id", "k"), ("aws_secret_access_key", "s")]);
let got = table_options(&unsigned, &fallback);
assert_eq!(got.get("aws_access_key_id").map(String::as_str), Some("k"));
let signed = described(&[("aws_endpoint", "http://s3:8333")]);
let got = table_options(&signed, &FallbackOptions::new());
assert!(!got.contains_key("aws_access_key_id"), "{got:?}");
}
#[test]
fn table_location_matches_the_gateway_layout() {
assert_eq!(
@@ -13,7 +13,7 @@ use seaweed_worker_core::pb::{
use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
use tracing::warn;
use crate::catalog::{NamespaceClient, parse_id};
use crate::catalog::{NamespaceClient, SigningKey, parse_id};
use crate::dataset;
use crate::jobs::{clamp, string_list, table_id};
@@ -32,6 +32,7 @@ const MAX_MIN_VERSIONS: i64 = 1000;
pub struct CleanupVersionsHandler {
namespace_url: String,
fallback: dataset::FallbackOptions,
signing_key: Option<SigningKey>,
metrics: Option<crate::metrics::LanceMetrics>,
}
@@ -40,6 +41,7 @@ impl CleanupVersionsHandler {
Self {
namespace_url,
fallback: dataset::FallbackOptions::new(),
signing_key: None,
metrics: None,
}
}
@@ -54,8 +56,15 @@ impl CleanupVersionsHandler {
self
}
pub fn with_signing_key(mut self, signing_key: Option<SigningKey>) -> Self {
self.signing_key = signing_key;
self
}
fn client(&self) -> NamespaceClient {
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
NamespaceClient::new(self.namespace_url.clone())
.with_fallback(&self.fallback)
.with_signing_key(self.signing_key.as_ref())
}
}
@@ -12,7 +12,7 @@ use seaweed_worker_core::pb::{
use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
use tracing::warn;
use crate::catalog::{NamespaceClient, parse_id};
use crate::catalog::{NamespaceClient, SigningKey, parse_id};
use crate::dataset;
use crate::jobs::{FORMAT, clamp, observation, string_list, table_id};
@@ -35,6 +35,7 @@ const MIN_FRAGMENTS_CEILING: i64 = 4096;
pub struct CompactHandler {
namespace_url: String,
fallback: dataset::FallbackOptions,
signing_key: Option<SigningKey>,
metrics: Option<crate::metrics::LanceMetrics>,
}
@@ -43,6 +44,7 @@ impl CompactHandler {
Self {
namespace_url,
fallback: dataset::FallbackOptions::new(),
signing_key: None,
metrics: None,
}
}
@@ -58,8 +60,15 @@ impl CompactHandler {
self
}
pub fn with_signing_key(mut self, signing_key: Option<SigningKey>) -> Self {
self.signing_key = signing_key;
self
}
fn client(&self) -> NamespaceClient {
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
NamespaceClient::new(self.namespace_url.clone())
.with_fallback(&self.fallback)
.with_signing_key(self.signing_key.as_ref())
}
}
@@ -13,7 +13,7 @@ use seaweed_worker_core::pb::{
use seaweed_worker_core::{DetectionSender, ExecutionSender, JobHandler};
use tracing::warn;
use crate::catalog::{NamespaceClient, parse_id};
use crate::catalog::{NamespaceClient, SigningKey, parse_id};
use crate::dataset::{self, OpenTable};
use crate::jobs::{clamp, string_list, table_id};
@@ -35,6 +35,7 @@ const MAX_UNINDEXED_CEILING: i64 = 100_000_000;
pub struct OptimizeIndicesHandler {
namespace_url: String,
fallback: dataset::FallbackOptions,
signing_key: Option<SigningKey>,
metrics: Option<crate::metrics::LanceMetrics>,
}
@@ -43,6 +44,7 @@ impl OptimizeIndicesHandler {
Self {
namespace_url,
fallback: dataset::FallbackOptions::new(),
signing_key: None,
metrics: None,
}
}
@@ -57,8 +59,15 @@ impl OptimizeIndicesHandler {
self
}
pub fn with_signing_key(mut self, signing_key: Option<SigningKey>) -> Self {
self.signing_key = signing_key;
self
}
fn client(&self) -> NamespaceClient {
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
NamespaceClient::new(self.namespace_url.clone())
.with_fallback(&self.fallback)
.with_signing_key(self.signing_key.as_ref())
}
}
@@ -42,27 +42,32 @@ pub(crate) fn table_id(parameters: &HashMap<String, ConfigValue>) -> Option<Vec<
pub fn handlers(
namespace_url: String,
fallback: crate::dataset::FallbackOptions,
signing_key: Option<crate::catalog::SigningKey>,
metrics: Option<crate::metrics::LanceMetrics>,
) -> Vec<Arc<dyn JobHandler>> {
vec![
Arc::new(
compact::CompactHandler::new(namespace_url.clone())
.with_fallback(fallback.clone())
.with_signing_key(signing_key.clone())
.with_metrics(metrics.clone()),
),
Arc::new(
indices::OptimizeIndicesHandler::new(namespace_url.clone())
.with_fallback(fallback.clone())
.with_signing_key(signing_key.clone())
.with_metrics(metrics.clone()),
),
Arc::new(
sort::SortHandler::new(namespace_url.clone())
.with_fallback(fallback.clone())
.with_signing_key(signing_key.clone())
.with_metrics(metrics.clone()),
),
Arc::new(
cleanup::CleanupVersionsHandler::new(namespace_url)
.with_fallback(fallback)
.with_signing_key(signing_key)
.with_metrics(metrics),
),
]
+11 -2
View File
@@ -23,7 +23,7 @@ use seaweed_worker_sort::{
};
use tracing::warn;
use crate::catalog::{NamespaceClient, parse_id};
use crate::catalog::{NamespaceClient, SigningKey, parse_id};
use crate::dataset;
use crate::jobs::{FORMAT, clamp, observation, string_list, table_id};
@@ -54,6 +54,7 @@ const BYTES_PER_MB: u64 = 1024 * 1024;
pub struct SortHandler {
namespace_url: String,
fallback: dataset::FallbackOptions,
signing_key: Option<SigningKey>,
metrics: Option<crate::metrics::LanceMetrics>,
}
@@ -62,6 +63,7 @@ impl SortHandler {
Self {
namespace_url,
fallback: dataset::FallbackOptions::new(),
signing_key: None,
metrics: None,
}
}
@@ -77,8 +79,15 @@ impl SortHandler {
self
}
pub fn with_signing_key(mut self, signing_key: Option<SigningKey>) -> Self {
self.signing_key = signing_key;
self
}
fn client(&self) -> NamespaceClient {
NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback)
NamespaceClient::new(self.namespace_url.clone())
.with_fallback(&self.fallback)
.with_signing_key(self.signing_key.as_ref())
}
}
+6 -3
View File
@@ -63,7 +63,8 @@ struct Args {
/// Shared secret for the admin control stream, the `jwt.filer_signing.key`
/// value the Go worker reads from its config file. Required by admins that
/// authenticate the worker stream with a signing key.
/// authenticate the worker stream with a signing key. The Lance namespace
/// takes it too: only a holder gets write credentials to vector indexes.
#[arg(long, env = "WEED_FILER_SIGNING_KEY")]
filer_signing_key: Option<String>,
@@ -136,6 +137,7 @@ async fn main() -> Result<()> {
let args = Args::parse();
let tls = args.tls()?;
let signing_key = weed_lance_worker::catalog::SigningKey::new(args.filer_signing_key.clone());
let options = WorkerOptions {
admin_address: args.admin,
worker_id: args.id,
@@ -164,9 +166,10 @@ async fn main() -> Result<()> {
}
let mut registry = Registry::new().with_preview(Arc::new(
weed_lance_worker::preview::LancePreview::new(args.namespace.clone(), fallback.clone()),
weed_lance_worker::preview::LancePreview::new(args.namespace.clone(), fallback.clone())
.with_signing_key(signing_key.clone()),
));
for handler in handlers(args.namespace, fallback, Some(lance_metrics)) {
for handler in handlers(args.namespace, fallback, signing_key, Some(lance_metrics)) {
registry = registry.register(handler);
}
+11 -2
View File
@@ -10,13 +10,14 @@ use async_trait::async_trait;
use futures::TryStreamExt;
use seaweed_worker_core::{Preview, PreviewProvider};
use crate::catalog::NamespaceClient;
use crate::catalog::{NamespaceClient, SigningKey};
use crate::dataset::{self, FallbackOptions};
use crate::jobs::FORMAT;
pub struct LancePreview {
namespace_url: String,
fallback: FallbackOptions,
signing_key: Option<SigningKey>,
}
impl LancePreview {
@@ -24,8 +25,14 @@ impl LancePreview {
Self {
namespace_url,
fallback,
signing_key: None,
}
}
pub fn with_signing_key(mut self, signing_key: Option<SigningKey>) -> Self {
self.signing_key = signing_key;
self
}
}
#[async_trait]
@@ -35,7 +42,9 @@ impl PreviewProvider for LancePreview {
}
async fn preview(&self, object_id: &[String], row_limit: usize) -> Result<Preview> {
let client = NamespaceClient::new(self.namespace_url.clone()).with_fallback(&self.fallback);
let client = NamespaceClient::new(self.namespace_url.clone())
.with_fallback(&self.fallback)
.with_signing_key(self.signing_key.as_ref());
let table = dataset::open(&client, object_id, &self.fallback).await?;
let total_rows = table.dataset.count_rows(None).await.unwrap_or(0) as i64;
+2 -1
View File
@@ -7,6 +7,7 @@ endif
VOLUME := ./weed-volume
CARGO_PROFILE ?= release
CARGO_TARGET_DIR ?= $(abspath ../../seaweed-vector/target)
override CARGO_TARGET_DIR := $(abspath $(CARGO_TARGET_DIR))
CARGO_OUTPUT := $(CARGO_TARGET_DIR)/$(if $(filter dev,$(CARGO_PROFILE)),debug,$(CARGO_PROFILE))
export CARGO_TARGET_DIR
@@ -18,7 +19,7 @@ $(WEED): FORCE
go build -o $(WEED) ../../weed
$(VEC): FORCE
cargo build --locked --profile $(CARGO_PROFILE) --manifest-path ../../seaweed-vector/Cargo.toml -p weed-vector-server
cd ../../seaweed-vector && cargo build --locked --profile $(CARGO_PROFILE) -p weed-vector-server
cp $(CARGO_OUTPUT)/weed-vector $(VEC)
ifneq ($(WORKER),)
+29 -9
View File
@@ -70,8 +70,18 @@ def stop(process):
@pytest.fixture(scope="module", params=combinations(), ids=lambda p: "-".join(p))
def cluster(request, tmp_path_factory):
volume, mode = request.param
directory = tmp_path_factory.mktemp(f"vectors-{volume}-{mode}")
yield from run_cluster(request.param, tmp_path_factory, identities=True)
@pytest.fixture(scope="module", params=combinations(), ids=lambda p: "-".join(p))
def open_cluster(request, tmp_path_factory):
"""A cluster with no S3 identities, and an engine given no credentials."""
yield from run_cluster(request.param, tmp_path_factory, identities=False)
def run_cluster(param, tmp_path_factory, identities):
volume, mode = param
directory = tmp_path_factory.mktemp(f"vectors-{volume}-{mode}" + ("" if identities else "-open"))
print(f"\nVector integration logs: {directory}")
weed = binary("WEED_BINARY", "weed")
engine = binary("VECTOR_BINARY", "weed-vector")
@@ -89,10 +99,14 @@ def cluster(request, tmp_path_factory):
master_addr = f"127.0.0.1:{master}.{master_grpc}"
filer_addr = f"127.0.0.1:{filer}.{filer_grpc}"
def start(name, args):
def start(name, args, unset=()):
log = stack.enter_context((directory / f"{name}.log").open("w"))
env = dict(os.environ, AWS_ACCESS_KEY_ID="test", AWS_SECRET_ACCESS_KEY="test",
AWS_DEFAULT_REGION="us-east-1", AWS_EC2_METADATA_DISABLED="true")
AWS_DEFAULT_REGION="us-east-1", AWS_EC2_METADATA_DISABLED="true",
WEED_S3_SSE_KEY="s3vectors-test")
env.pop("WEED_S3_SSE_KEK", None)
for key in unset:
env.pop(key)
process = subprocess.Popen(args, cwd=directory, stdout=log, stderr=log, env=env)
processes[name] = process
stack.callback(stop, process)
@@ -117,14 +131,20 @@ def cluster(request, tmp_path_factory):
f"-defaultStoreDir={directory / 'filer'}"])
wait_for(lambda: listening(filer_grpc), processes, "filer not ready")
endpoint = f"http://127.0.0.1:{s3}"
start("s3", [weed, "-v=1", "s3", f"-filer={filer_addr}", f"-config={HERE / 's3.json'}",
# weed s3 turns AWS_ACCESS_KEY_ID into an admin identity, and the engine
# would find it as a credential.
if identities:
config, unset = [f"-config={HERE / 's3.json'}"], ()
credentials = ["--storage-option", "aws_access_key_id=test",
"--storage-option", "aws_secret_access_key=test"]
else:
config, unset, credentials = [], ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"), []
start("s3", [weed, "-v=1", "s3", f"-filer={filer_addr}", *config,
"-ip=127.0.0.1", "-ip.bind=127.0.0.1", f"-port={s3}",
f"-port.grpc={s3_grpc}", "-port.iceberg=0", f"-port.lance={lance}"])
f"-port.grpc={s3_grpc}", "-port.iceberg=0", f"-port.lance={lance}"], unset)
wait_for(lambda: http_ready(f"http://127.0.0.1:{lance}/v1/namespace/root/list"), processes, "catalog not ready")
start("vector", [engine, "--namespace", f"http://127.0.0.1:{lance}",
"--masters", master_addr, "--listen", f"127.0.0.1:{vector}",
"--storage-option", "aws_access_key_id=test",
"--storage-option", "aws_secret_access_key=test"])
"--masters", master_addr, "--listen", f"127.0.0.1:{vector}", *credentials], unset)
wait_for(lambda: listening(vector), processes, "vector engine not ready")
wait_for(
+106 -2
View File
@@ -1,10 +1,12 @@
"""Run the S3 Vectors API against Go/Rust volumes and single/distributed queries."""
import contextlib
import json
import time
import uuid
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
import pytest
@@ -58,7 +60,7 @@ def test_vector_lifecycle(vectors):
assert [h["key"] for h in hits] == ["a", "c"]
distances = {h["key"]: h["distance"] for h in hits}
assert distances["a"] == pytest.approx(0.0, abs=1e-6)
assert distances["c"] == pytest.approx(0.02, abs=1e-4)
assert distances["c"] == pytest.approx(0.02 ** 0.5, abs=1e-4)
assert all(h["metadata"]["color"] == "red" for h in hits)
vectors.delete_vectors(vectorBucketName=BUCKET, indexName=INDEX, keys=["b"])
@@ -204,6 +206,32 @@ def test_filter_types(vectors, index):
assert sorted(hit["key"] for hit in hits) == ["0", "3"]
def test_filter_arrays(vectors, index):
rows = {"space": ["space", "classic"], "nums": [1, 2.5], "strs": ["1", "2.5"], "flag": [True], "scalar": "space"}
vectors.put_vectors(**index, vectors=[{"key": key, "data": {"float32": [1, 0, 0, 0]}, "metadata": {"tags": tags}}
for key, tags in rows.items()])
def matches(condition):
hits = vectors.query_vectors(**index, queryVector={"float32": [1, 0, 0, 0]}, topK=10,
filter={"tags": condition})["vectors"]
return sorted(hit["key"] for hit in hits)
assert matches("space") == ["scalar", "space"]
assert matches({"$eq": 1}) == ["nums"]
assert matches({"$eq": "1"}) == ["strs"]
assert matches({"$eq": True}) == ["flag"]
assert matches({"$in": ["classic", 2.5]}) == ["nums", "space"]
assert matches({"$ne": "space"}) == ["flag", "nums", "strs"]
assert matches({"$nin": [1, "2.5"]}) == ["flag", "scalar", "space"]
def test_euclidean_distance(vectors, index):
vectors.put_vectors(**index, vectors=[{"key": "p", "data": {"float32": [3, 4, 0, 0]}}])
result = vectors.query_vectors(**index, queryVector={"float32": [0, 0, 0, 0]}, topK=1, returnDistance=True)
assert result["distanceMetric"] == "euclidean"
assert result["vectors"][0]["distance"] == pytest.approx(5.0)
def test_cosine_metric(vectors):
bucket = "cosine-" + uuid.uuid4().hex[:16]
vectors.create_vector_bucket(vectorBucketName=bucket)
@@ -270,17 +298,44 @@ def test_vector_policy_permissions(vectors, vector_clients, index):
vectors.delete_index(**other)
def index_object_encryption(vectors, bucket):
s3 = boto3.client("s3", endpoint_url=vectors.meta.endpoint_url, aws_access_key_id="test",
aws_secret_access_key="test", region_name="us-east-1")
keys = [o["Key"] for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix=f"vector/{INDEX}/")
for o in page.get("Contents", []) if o["Size"] > 0]
assert any("/data/" in key for key in keys) and any("/_versions/" in key for key in keys)
return {key: s3.head_object(Bucket=bucket, Key=key).get("ServerSideEncryption") for key in keys}
def test_vector_bucket_encryption(vectors):
bucket = "encryption-" + uuid.uuid4().hex[:16]
plain = bucket + "-plain"
config = {"sseType": "AES256"}
vectors.create_vector_bucket(vectorBucketName=bucket, encryptionConfiguration=config)
vectors.create_vector_bucket(vectorBucketName=plain)
try:
assert vectors.get_vector_bucket(vectorBucketName=bucket)["vectorBucket"]["encryptionConfiguration"] == config
expect_error("ValidationException", vectors.create_vector_bucket, vectorBucketName=bucket + "-kms",
encryptionConfiguration={"sseType": "aws:kms", "kmsKeyArn": "arn:aws:kms:us-east-1:123456789012:key/test"})
assert vectors.list_vector_buckets(prefix=bucket + "-kms")["vectorBuckets"] == []
for name in (bucket, plain):
vectors.create_index(vectorBucketName=name, indexName=INDEX, dataType="float32", dimension=4,
distanceMetric="euclidean")
vectors.put_vectors(vectorBucketName=name, indexName=INDEX, vectors=[
{"key": "a", "data": {"float32": [1, 0, 0, 0]}, "metadata": {"color": "red"}},
{"key": "b", "data": {"float32": [0, 1, 0, 0]}, "metadata": {"color": "blue"}},
])
hits = vectors.query_vectors(vectorBucketName=name, indexName=INDEX, queryVector={"float32": [1, 0, 0, 0]},
topK=1, returnMetadata=True)["vectors"]
assert [(h["key"], h["metadata"]) for h in hits] == [("a", {"color": "red"})]
assert set(index_object_encryption(vectors, bucket).values()) == {"AES256"}
assert set(index_object_encryption(vectors, plain).values()) == {None}
finally:
vectors.delete_vector_bucket(vectorBucketName=bucket)
for name in (bucket, plain):
with contextlib.suppress(ClientError):
vectors.delete_index(vectorBucketName=name, indexName=INDEX)
vectors.delete_vector_bucket(vectorBucketName=name)
def test_catalog_pagination(vectors, index):
@@ -362,6 +417,40 @@ def test_index_arn_validation(vectors, index):
assert vectors.get_index(**index)["index"]["indexArn"] == arn
def test_resource_tags(vectors, vector_clients, index):
reader = vector_clients["reader"]
bucket_arn = vectors.get_vector_bucket(vectorBucketName=index["vectorBucketName"])["vectorBucket"]["vectorBucketArn"]
index_arn = vectors.get_index(**index)["index"]["indexArn"]
assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {}
vectors.tag_resource(resourceArn=bucket_arn, tags={"team": "search", "env": "dev"})
vectors.tag_resource(resourceArn=index_arn, tags={"team": "ranking"})
assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {"team": "search", "env": "dev"}
assert vectors.list_tags_for_resource(resourceArn=index_arn)["tags"] == {"team": "ranking"}
vectors.untag_resource(resourceArn=bucket_arn, tagKeys=["env", "absent"])
assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {"team": "search"}
expect_error("ValidationException", vectors.tag_resource, resourceArn=bucket_arn,
tags={f"k{i}": "v" for i in range(50)})
expect_error("NotFoundException", vectors.list_tags_for_resource, resourceArn=bucket_arn + "/index/missing")
expect_error("AccessDeniedException", reader.list_tags_for_resource, resourceArn=bucket_arn)
expect_error("AccessDeniedException", reader.tag_resource, resourceArn=index_arn, tags={"team": "other"})
assert vectors.list_tags_for_resource(resourceArn=index_arn)["tags"] == {"team": "ranking"}
def test_create_with_tags(vectors):
bucket = "tagged-" + uuid.uuid4().hex[:16]
bucket_arn = vectors.create_vector_bucket(vectorBucketName=bucket, tags={"team": "search"})["vectorBucketArn"]
try:
assert vectors.list_tags_for_resource(resourceArn=bucket_arn)["tags"] == {"team": "search"}
index_arn = vectors.create_index(vectorBucketName=bucket, indexName=INDEX, dataType="float32", dimension=4,
distanceMetric="cosine", tags={"tier": "hot"})["indexArn"]
try:
assert vectors.list_tags_for_resource(resourceArn=index_arn)["tags"] == {"tier": "hot"}
finally:
vectors.delete_index(vectorBucketName=bucket, indexName=INDEX)
finally:
vectors.delete_vector_bucket(vectorBucketName=bucket)
def test_invalid_credentials(cluster):
client = boto3.client("s3vectors", endpoint_url=cluster["endpoint"], aws_access_key_id="unknown",
aws_secret_access_key="unknown", region_name="us-east-1")
@@ -389,3 +478,18 @@ def test_query_without_writer(vectors, index, cluster):
raise
time.sleep(0.2)
assert [v["key"] for v in result["vectors"]] == ["survivor"]
def test_engine_without_credentials(open_cluster):
"""With no S3 identities, an engine given no credentials still serves an index."""
client = boto3.client("s3vectors", endpoint_url=open_cluster["endpoint"], aws_access_key_id="any",
aws_secret_access_key="any", region_name="us-east-1",
config=Config(connect_timeout=3, read_timeout=20, retries={"max_attempts": 0}))
index = {"vectorBucketName": BUCKET, "indexName": INDEX}
client.create_vector_bucket(vectorBucketName=BUCKET)
client.create_index(**index, dataType="float32", dimension=2, distanceMetric="cosine")
client.put_vectors(**index, vectors=[{"key": "a", "data": {"float32": [1, 0]}},
{"key": "b", "data": {"float32": [0, 1]}}])
hits = client.query_vectors(**index, queryVector={"float32": [1, 0]}, topK=1)["vectors"]
assert [h["key"] for h in hits] == ["a"]
+23
View File
@@ -43,6 +43,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/scheduler"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
_ "github.com/seaweedfs/seaweedfs/weed/credential/grpc" // Register gRPC credential store
@@ -167,14 +168,21 @@ type AdminServer struct {
collectionStatsCacheThreshold time.Duration
s3TablesManager *s3tables.Manager
s3VectorsServer *s3vectors.Server
icebergPort int
lancePort int
// s3PublicEndpoint is the client-facing S3 address from admin.toml
// (s3.public_endpoint); discovered S3 servers are the fallback.
s3PublicEndpoint string
// uploadMaxSize caps the total bytes of one file browser upload; 0 means no limit.
uploadMaxSize int64
}
// DefaultUploadMaxSizeMB is the default total size limit of one file browser upload.
const DefaultUploadMaxSizeMB = 500
// Type definitions moved to types.go
func NewAdminServer(masters string, filerGroup string, templateFS http.FileSystem, dataDir string, icebergPort, lancePort int, s3PublicEndpoint string) *AdminServer {
@@ -215,11 +223,14 @@ func NewAdminServer(masters string, filerGroup string, templateFS http.FileSyste
icebergPort: icebergPort,
lancePort: lancePort,
s3PublicEndpoint: normalizeS3PublicEndpoint(s3PublicEndpoint),
uploadMaxSize: DefaultUploadMaxSizeMB << 20,
pluginLock: lockManager,
adminPresenceLock: presenceLock,
bgCancel: bgCancel,
}
server.s3VectorsServer = newS3VectorsServer(server)
// Initialize topic retention purger
server.topicRetentionPurger = NewTopicRetentionPurger(server)
@@ -1570,6 +1581,18 @@ func (s *AdminServer) GetS3Endpoint() string {
return s.s3PublicEndpoint
}
// SetUploadMaxSize sets the total size limit in bytes of one file browser
// upload; zero or less removes the limit.
func (s *AdminServer) SetUploadMaxSize(maxBytes int64) {
s.uploadMaxSize = max(maxBytes, 0)
}
// GetUploadMaxSize returns the total size limit in bytes of one file browser
// upload, or 0 when uploads are unlimited.
func (s *AdminServer) GetUploadMaxSize() int64 {
return s.uploadMaxSize
}
// normalizeS3PublicEndpoint trims a trailing slash and drops, with a warning,
// a value that is not a plain absolute http or https URL, so the file browser
// hides its URL actions instead of copying broken links. Checking the string
+4
View File
@@ -53,6 +53,8 @@ type FileBrowserData struct {
HasNextPage bool `json:"has_next_page"`
LastFileName string `json:"last_file_name"` // Cursor for next page
CurrentLastFileName string `json:"current_last_file_name"` // Cursor from current request (for page size changes)
UploadMaxSize int64 `json:"upload_max_size"`
}
// GetFileBrowser retrieves file browser data for a given path with cursor-based
@@ -240,6 +242,8 @@ func (s *AdminServer) GetFileBrowser(dir string, prefix string, lastFileName str
HasNextPage: hasNextPage,
LastFileName: lastEntryName, // Store for next page navigation
CurrentLastFileName: lastFileName, // Store input cursor for page size changes
UploadMaxSize: s.uploadMaxSize,
}, nil
}
+254
View File
@@ -0,0 +1,254 @@
package dash
import (
"context"
"errors"
"net/http"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
)
type S3VectorsBucketsData struct {
Username string `json:"username"`
Buckets []S3VectorsBucketSummary `json:"buckets"`
TotalBuckets int `json:"total_buckets"`
Truncated bool `json:"truncated,omitempty"`
LastUpdated time.Time `json:"last_updated"`
}
type S3VectorsBucketSummary struct {
Name string `json:"name"`
ARN string `json:"arn"`
CreatedAt time.Time `json:"createdAt"`
// Encryption is the bucket's sseType, or empty when it declares none.
Encryption string `json:"encryption,omitempty"`
}
type S3VectorsBucketData struct {
Username string `json:"username"`
Bucket S3VectorsBucketSummary `json:"bucket"`
Policy string `json:"policy,omitempty"`
Indexes []*s3vectors.Index `json:"indexes"`
TotalIndexes int `json:"total_indexes"`
Truncated bool `json:"truncated,omitempty"`
LastUpdated time.Time `json:"last_updated"`
}
// The S3 Vectors operations reach the filer through the admin server's own
// filer connection.
type s3VectorsFilerClient struct{ s *AdminServer }
func (c s3VectorsFilerClient) WithFilerClient(_ bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
return c.s.WithFilerClient(fn)
}
// newS3VectorsServer serves the catalog side of S3 Vectors for the admin
// console. It has no vector engine, so it never runs an operation that needs
// one, such as CreateIndex.
func newS3VectorsServer(s *AdminServer) *s3vectors.Server {
server := s3vectors.NewServer(s3VectorsFilerClient{s}, nil, nil)
server.SetRegion(s3tables.DefaultRegion)
server.SetAccountID(s3tables.DefaultAccountID)
return server
}
func (s *AdminServer) executeS3VectorsOperation(ctx context.Context, operation string, req, resp interface{}) error {
return s.s3VectorsServer.Execute(ctx, operation, req, resp, s3_constants.AccountAdminId)
}
const s3VectorsAdminPage = 500
func (s *AdminServer) s3VectorsBucketSummary(ctx context.Context, name string) (S3VectorsBucketSummary, error) {
var resp s3vectors.GetVectorBucketResponse
if err := s.executeS3VectorsOperation(ctx, "GetVectorBucket", &s3vectors.GetVectorBucketRequest{VectorBucketName: name}, &resp); err != nil {
return S3VectorsBucketSummary{}, err
}
b := resp.VectorBucket
summary := S3VectorsBucketSummary{Name: b.VectorBucketName, ARN: b.VectorBucketArn, CreatedAt: b.CreationTime}
if sse, ok := b.EncryptionConfiguration["sseType"].(string); ok {
summary.Encryption = sse
}
return summary, nil
}
func (s *AdminServer) GetS3VectorsBucketsData(ctx context.Context) (S3VectorsBucketsData, error) {
data := S3VectorsBucketsData{Buckets: []S3VectorsBucketSummary{}}
page := s3VectorsAdminPage
req := &s3vectors.ListVectorBucketsRequest{MaxResults: &page}
for {
var resp s3vectors.ListVectorBucketsResponse
if err := s.executeS3VectorsOperation(ctx, "ListVectorBuckets", req, &resp); err != nil {
return S3VectorsBucketsData{}, err
}
for _, b := range resp.VectorBuckets {
if len(data.Buckets) == s3TablesAdminListLimit {
data.Truncated = true
break
}
summary, err := s.s3VectorsBucketSummary(ctx, b.VectorBucketName)
if isS3VectorsNotFound(err) {
continue
}
if err != nil {
return S3VectorsBucketsData{}, err
}
data.Buckets = append(data.Buckets, summary)
}
if data.Truncated || resp.NextToken == "" {
break
}
req.NextToken = resp.NextToken
}
data.TotalBuckets = len(data.Buckets)
data.LastUpdated = time.Now()
return data, nil
}
func (s *AdminServer) GetS3VectorsBucketData(ctx context.Context, bucket string) (S3VectorsBucketData, error) {
summary, err := s.s3VectorsBucketSummary(ctx, bucket)
if err != nil {
return S3VectorsBucketData{}, err
}
data := S3VectorsBucketData{Bucket: summary, Indexes: []*s3vectors.Index{}, LastUpdated: time.Now()}
var policy s3vectors.GetVectorBucketPolicyResponse
switch err := s.executeS3VectorsOperation(ctx, "GetVectorBucketPolicy", &s3vectors.GetVectorBucketPolicyRequest{VectorBucketName: bucket}, &policy); {
case err == nil:
data.Policy = policy.Policy
case !isS3VectorsNotFound(err):
return S3VectorsBucketData{}, err
}
page := s3VectorsAdminPage
req := &s3vectors.ListIndexesRequest{VectorBucketName: bucket, MaxResults: &page}
for {
var resp s3vectors.ListIndexesResponse
if err := s.executeS3VectorsOperation(ctx, "ListIndexes", req, &resp); err != nil {
return S3VectorsBucketData{}, err
}
for _, summary := range resp.Indexes {
if len(data.Indexes) == s3TablesAdminListLimit {
data.Truncated = true
break
}
var index s3vectors.GetIndexResponse
err := s.executeS3VectorsOperation(ctx, "GetIndex", &s3vectors.GetIndexRequest{VectorBucketName: bucket, IndexName: summary.IndexName}, &index)
if isS3VectorsNotFound(err) {
continue
}
if err != nil {
return S3VectorsBucketData{}, err
}
data.Indexes = append(data.Indexes, index.Index)
}
if data.Truncated || resp.NextToken == "" {
break
}
req.NextToken = resp.NextToken
}
data.TotalIndexes = len(data.Indexes)
return data, nil
}
func (s *AdminServer) ListS3VectorsBucketsAPI(w http.ResponseWriter, r *http.Request) {
data, err := s.GetS3VectorsBucketsData(r.Context())
if err != nil {
writeS3VectorsError(w, err)
return
}
writeJSON(w, http.StatusOK, data)
}
func (s *AdminServer) CreateS3VectorsBucket(w http.ResponseWriter, r *http.Request) {
if !requireSessionCSRFToken(w, r) {
return
}
var req struct {
Name string `json:"name"`
}
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
return
}
var resp s3vectors.CreateVectorBucketResponse
if err := s.executeS3VectorsOperation(r.Context(), "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: req.Name}, &resp); err != nil {
writeS3VectorsError(w, err)
return
}
writeJSON(w, http.StatusCreated, map[string]interface{}{"arn": resp.VectorBucketArn})
}
func (s *AdminServer) DeleteS3VectorsBucket(w http.ResponseWriter, r *http.Request) {
if !requireSessionCSRFToken(w, r) {
return
}
bucket := r.URL.Query().Get("bucket")
if bucket == "" {
writeJSONError(w, http.StatusBadRequest, "bucket query parameter is required")
return
}
if err := s.executeS3VectorsOperation(r.Context(), "DeleteVectorBucket", &s3vectors.DeleteVectorBucketRequest{VectorBucketName: bucket}, nil); err != nil {
writeS3VectorsError(w, err)
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Vector bucket deleted"})
}
func (s *AdminServer) GetS3VectorsBucketAPI(w http.ResponseWriter, r *http.Request) {
bucket := r.URL.Query().Get("bucket")
if bucket == "" {
writeJSONError(w, http.StatusBadRequest, "bucket query parameter is required")
return
}
data, err := s.GetS3VectorsBucketData(r.Context(), bucket)
if err != nil {
writeS3VectorsError(w, err)
return
}
writeJSON(w, http.StatusOK, data)
}
func (s *AdminServer) DeleteS3VectorsIndex(w http.ResponseWriter, r *http.Request) {
if !requireSessionCSRFToken(w, r) {
return
}
bucket, index := r.URL.Query().Get("bucket"), r.URL.Query().Get("index")
if bucket == "" || index == "" {
writeJSONError(w, http.StatusBadRequest, "bucket and index query parameters are required")
return
}
if err := s.executeS3VectorsOperation(r.Context(), "DeleteIndex", &s3vectors.DeleteIndexRequest{VectorBucketName: bucket, IndexName: index}, nil); err != nil {
writeS3VectorsError(w, err)
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{"message": "Index deleted"})
}
func isS3VectorsNotFound(err error) bool {
var e *s3vectors.Error
return errors.As(err, &e) && e.Code == "NotFoundException"
}
func writeS3VectorsError(w http.ResponseWriter, err error) {
var e *s3vectors.Error
if !errors.As(err, &e) {
writeJSONError(w, http.StatusInternalServerError, err.Error())
return
}
status := http.StatusInternalServerError
switch e.Code {
case "ValidationException":
status = http.StatusBadRequest
case "AccessDeniedException":
status = http.StatusForbidden
case "NotFoundException":
status = http.StatusNotFound
case "ConflictException":
status = http.StatusConflict
case "ServiceUnavailableException":
status = http.StatusServiceUnavailable
}
writeJSONError(w, status, e.Message)
}
@@ -0,0 +1,174 @@
package dash
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
)
func newS3VectorsTestServer(t *testing.T) (*AdminServer, *s3tables.ManagerClient) {
t.Helper()
fs := s3tablestest.Start(t)
client := s3tables.NewManagerClient(fs.Client)
server := s3vectors.NewServer(client, nil, nil)
server.SetRegion(s3tables.DefaultRegion)
server.SetAccountID(s3tables.DefaultAccountID)
return &AdminServer{s3VectorsServer: server}, client
}
// createS3VectorsIndexRecord writes the catalog record CreateIndex leaves
// behind; the admin server has no engine to run CreateIndex itself.
func createS3VectorsIndexRecord(t *testing.T, client *s3tables.ManagerClient, bucket, index string, metadata map[string]string) {
t.Helper()
m := s3tables.NewManager()
m.SetTrusted(true)
arn := "arn:aws:s3tables:us-east-1:000000000000:bucket/" + bucket
ctx := context.Background()
err := m.Execute(ctx, client, "CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}}, nil, s3_constants.AccountAdminId)
if err != nil && !strings.Contains(err.Error(), "already exists") {
t.Fatalf("create namespace: %v", err)
}
full, _ := json.Marshal(metadata)
if err := m.Execute(ctx, client, "CreateTable", &s3tables.CreateTableRequest{
TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}, Name: index,
Format: s3tables.FormatLance, Metadata: &s3tables.TableMetadata{FullMetadata: full},
}, nil, s3_constants.AccountAdminId); err != nil {
t.Fatalf("create index record: %v", err)
}
}
func s3VectorsRequest(t *testing.T, handler http.HandlerFunc, method, target, body string) *httptest.ResponseRecorder {
t.Helper()
w := httptest.NewRecorder()
handler(w, httptest.NewRequest(method, target, strings.NewReader(body)))
return w
}
func TestS3VectorsBucketsListsOnlyVectorBuckets(t *testing.T) {
s, client := newS3VectorsTestServer(t)
ctx := context.Background()
if w := s3VectorsRequest(t, s.CreateS3VectorsBucket, http.MethodPost, "/api/s3vectors/buckets", `{"name":"media"}`); w.Code != http.StatusCreated {
t.Fatalf("create: %d %s", w.Code, w.Body)
}
if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{
VectorBucketName: "sealed", EncryptionConfiguration: map[string]interface{}{"sseType": "AES256"},
}, nil); err != nil {
t.Fatalf("create sealed: %v", err)
}
if err := s3tables.NewManager().Execute(ctx, client, "CreateTableBucket", &s3tables.CreateTableBucketRequest{
Name: "warehouse", Format: s3tables.FormatIceberg,
}, nil, s3_constants.AccountAdminId); err != nil {
t.Fatalf("create iceberg bucket: %v", err)
}
data, err := s.GetS3VectorsBucketsData(ctx)
if err != nil {
t.Fatal(err)
}
got := map[string]string{}
for _, b := range data.Buckets {
got[b.Name] = b.Encryption
if want := "arn:aws:s3vectors:us-east-1:000000000000:bucket/" + b.Name; b.ARN != want || b.CreatedAt.IsZero() {
t.Errorf("%s: arn %q created %v, want %q and a time", b.Name, b.ARN, b.CreatedAt, want)
}
}
if len(got) != 2 || got["media"] != "" || got["sealed"] != "AES256" || data.TotalBuckets != 2 || data.Truncated {
t.Fatalf("buckets %v (total %d), want media and sealed (AES256) without the iceberg bucket", got, data.TotalBuckets)
}
}
func TestS3VectorsBucketShowsIndexesAndPolicy(t *testing.T) {
s, client := newS3VectorsTestServer(t)
ctx := context.Background()
if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: "media"}, nil); err != nil {
t.Fatal(err)
}
createS3VectorsIndexRecord(t, client, "media", "movies", map[string]string{
"dimension": "3", "distanceMetric": "cosine", "nonFilterableMetadataKeys": `["summary"]`,
})
policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3vectors:QueryVectors","Resource":"*"}]}`
if err := s.executeS3VectorsOperation(ctx, "PutVectorBucketPolicy", &s3vectors.PutVectorBucketPolicyRequest{VectorBucketName: "media", Policy: policy}, nil); err != nil {
t.Fatal(err)
}
w := s3VectorsRequest(t, s.GetS3VectorsBucketAPI, http.MethodGet, "/api/s3vectors/indexes?bucket=media", "")
if w.Code != http.StatusOK {
t.Fatalf("get: %d %s", w.Code, w.Body)
}
var data S3VectorsBucketData
if err := json.Unmarshal(w.Body.Bytes(), &data); err != nil {
t.Fatal(err)
}
if data.Policy != policy {
t.Errorf("policy %q, want %q", data.Policy, policy)
}
if data.TotalIndexes != 1 {
t.Fatalf("indexes %+v", data.Indexes)
}
idx := data.Indexes[0]
if idx.IndexName != "movies" || idx.Dimension != 3 || idx.DistanceMetric != "cosine" || idx.DataType != "float32" ||
idx.MetadataConfiguration == nil || len(idx.MetadataConfiguration.NonFilterableMetadataKeys) != 1 {
t.Fatalf("index %+v", idx)
}
if w := s3VectorsRequest(t, s.GetS3VectorsBucketAPI, http.MethodGet, "/api/s3vectors/indexes?bucket=missing", ""); w.Code != http.StatusNotFound {
t.Fatalf("missing bucket: %d, want 404", w.Code)
}
}
func TestS3VectorsDeleteIndexThenBucket(t *testing.T) {
s, client := newS3VectorsTestServer(t)
ctx := context.Background()
if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: "media"}, nil); err != nil {
t.Fatal(err)
}
createS3VectorsIndexRecord(t, client, "media", "movies", map[string]string{"dimension": "3", "distanceMetric": "cosine"})
if w := s3VectorsRequest(t, s.DeleteS3VectorsBucket, http.MethodDelete, "/api/s3vectors/buckets?bucket=media", ""); w.Code != http.StatusConflict {
t.Fatalf("delete a bucket with an index: %d %s, want 409", w.Code, w.Body)
}
if w := s3VectorsRequest(t, s.DeleteS3VectorsIndex, http.MethodDelete, "/api/s3vectors/indexes?bucket=media&index=movies", ""); w.Code != http.StatusOK {
t.Fatalf("delete index: %d %s", w.Code, w.Body)
}
// The emptied index namespace does not keep the bucket alive.
if w := s3VectorsRequest(t, s.DeleteS3VectorsBucket, http.MethodDelete, "/api/s3vectors/buckets?bucket=media", ""); w.Code != http.StatusOK {
t.Fatalf("delete bucket: %d %s", w.Code, w.Body)
}
data, err := s.GetS3VectorsBucketsData(ctx)
if err != nil || data.TotalBuckets != 0 {
t.Fatalf("after delete: %+v, %v", data.Buckets, err)
}
}
func TestS3VectorsBucketsStopAtTheListLimit(t *testing.T) {
s, client := newS3VectorsTestServer(t)
ctx := context.Background()
// The ICEBERG bucket leaves the first page short, so the last page
// would carry the list past the limit.
if err := s3tables.NewManager().Execute(ctx, client, "CreateTableBucket", &s3tables.CreateTableBucketRequest{
Name: "iceberg", Format: s3tables.FormatIceberg,
}, nil, s3_constants.AccountAdminId); err != nil {
t.Fatal(err)
}
for i := 0; i <= s3TablesAdminListLimit; i++ {
if err := s.executeS3VectorsOperation(ctx, "CreateVectorBucket", &s3vectors.CreateVectorBucketRequest{VectorBucketName: fmt.Sprintf("media-%04d", i)}, nil); err != nil {
t.Fatal(err)
}
}
data, err := s.GetS3VectorsBucketsData(ctx)
if err != nil {
t.Fatal(err)
}
if data.TotalBuckets != s3TablesAdminListLimit || !data.Truncated {
t.Fatalf("listed %d buckets (truncated %v), want the limit %d and truncated", data.TotalBuckets, data.Truncated, s3TablesAdminListLimit)
}
}
+50
View File
@@ -1,6 +1,7 @@
package handlers
import (
"errors"
"net/http"
"net/url"
"strconv"
@@ -14,6 +15,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
"github.com/seaweedfs/seaweedfs/weed/stats"
)
@@ -111,6 +113,8 @@ func (h *AdminHandlers) registerUIRoutes(r *mux.Router) {
r.HandleFunc("/object-store/groups", h.groupHandlers.ShowGroups).Methods(http.MethodGet)
r.HandleFunc("/object-store/service-accounts", h.serviceAccountHandlers.ShowServiceAccounts).Methods(http.MethodGet)
r.HandleFunc("/object-store/s3tables/buckets", h.ShowS3TablesBuckets).Methods(http.MethodGet)
r.HandleFunc("/object-store/vector-buckets", h.ShowS3VectorsBuckets).Methods(http.MethodGet)
r.HandleFunc("/object-store/vector-buckets/{bucket}", h.ShowS3VectorsBucketDetails).Methods(http.MethodGet)
r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces", h.ShowS3TablesNamespaces).Methods(http.MethodGet)
r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces/{namespace}/tables", h.ShowS3TablesTables).Methods(http.MethodGet)
r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces/{namespace}/tables/{table}", h.ShowS3TablesTableDetails).Methods(http.MethodGet)
@@ -251,6 +255,13 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
s3TablesApi.HandleFunc("/tags", h.adminServer.ListS3TablesTags).Methods(http.MethodGet)
s3TablesApi.Handle("/tags", wrapWrite(h.adminServer.UntagS3TablesResource)).Methods(http.MethodDelete)
s3VectorsApi := api.PathPrefix("/s3vectors").Subrouter()
s3VectorsApi.HandleFunc("/buckets", h.adminServer.ListS3VectorsBucketsAPI).Methods(http.MethodGet)
s3VectorsApi.Handle("/buckets", wrapWrite(h.adminServer.CreateS3VectorsBucket)).Methods(http.MethodPost)
s3VectorsApi.Handle("/buckets", wrapWrite(h.adminServer.DeleteS3VectorsBucket)).Methods(http.MethodDelete)
s3VectorsApi.HandleFunc("/indexes", h.adminServer.GetS3VectorsBucketAPI).Methods(http.MethodGet)
s3VectorsApi.Handle("/indexes", wrapWrite(h.adminServer.DeleteS3VectorsIndex)).Methods(http.MethodDelete)
filesApi := api.PathPrefix("/files").Subrouter()
filesApi.Handle("/delete", wrapWrite(h.fileBrowserHandlers.DeleteFile)).Methods(http.MethodDelete)
filesApi.Handle("/delete-multiple", wrapWrite(h.fileBrowserHandlers.DeleteMultipleFiles)).Methods(http.MethodDelete)
@@ -373,6 +384,45 @@ func (h *AdminHandlers) ShowS3TablesBuckets(w http.ResponseWriter, r *http.Reque
}
}
// ShowS3VectorsBuckets renders the vector buckets page
func (h *AdminHandlers) ShowS3VectorsBuckets(w http.ResponseWriter, r *http.Request) {
username := h.getUsername(r)
data, err := h.adminServer.GetS3VectorsBucketsData(r.Context())
if err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to get vector buckets: "+err.Error())
return
}
data.Username = username
w.Header().Set("Content-Type", "text/html")
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
if err := layout.Layout(viewCtx, app.S3VectorsBuckets(data)).Render(r.Context(), w); err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
}
}
// ShowS3VectorsBucketDetails renders one vector bucket with its indexes and policy
func (h *AdminHandlers) ShowS3VectorsBucketDetails(w http.ResponseWriter, r *http.Request) {
username := h.getUsername(r)
data, err := h.adminServer.GetS3VectorsBucketData(r.Context(), mux.Vars(r)["bucket"])
if err != nil {
status := http.StatusInternalServerError
var e *s3vectors.Error
if errors.As(err, &e) && e.Code == "NotFoundException" {
status = http.StatusNotFound
}
writeJSONError(w, status, "Failed to get vector bucket: "+err.Error())
return
}
data.Username = username
w.Header().Set("Content-Type", "text/html")
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
if err := layout.Layout(viewCtx, app.S3VectorsBucketDetails(data)).Render(r.Context(), w); err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
}
}
// ShowS3TablesNamespaces renders namespaces for a table bucket
func (h *AdminHandlers) ShowS3TablesNamespaces(w http.ResponseWriter, r *http.Request) {
username := h.getUsername(r)
+39 -8
View File
@@ -4,6 +4,7 @@ import (
"bufio"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"mime"
@@ -247,25 +248,46 @@ func (h *FileBrowserHandlers) CreateFolder(w http.ResponseWriter, r *http.Reques
// UploadFile handles file upload requests
func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request) {
// Get the current path
maxSize := h.adminServer.GetUploadMaxSize()
if maxSize > 0 {
r.Body = http.MaxBytesReader(w, r.Body, maxSize+uploadFormOverhead)
}
// Parts beyond 32MB spill to temporary files instead of memory
err := r.ParseMultipartForm(32 << 20)
if err != nil {
var maxBytesErr *http.MaxBytesError
if errors.As(err, &maxBytesErr) {
writeJSONError(w, http.StatusRequestEntityTooLarge, uploadTooLargeMessage(maxSize))
return
}
writeJSONError(w, http.StatusBadRequest, "Failed to parse multipart form: "+err.Error())
return
}
// The router hands us a copy of the request, so the server cannot remove the spooled parts itself
defer r.MultipartForm.RemoveAll()
currentPath := r.FormValue("path")
if currentPath == "" {
currentPath = "/"
}
// Parse multipart form
err := r.ParseMultipartForm(1 << 30) // 1GB max memory for large file uploads
if err != nil {
writeJSONError(w, http.StatusBadRequest, "Failed to parse multipart form: "+err.Error())
return
}
// Get uploaded files (supports multiple files)
files := r.MultipartForm.File["files"]
if len(files) == 0 {
writeJSONError(w, http.StatusBadRequest, "No files uploaded")
return
}
if maxSize > 0 {
var totalSize int64
for _, fileHeader := range files {
totalSize += fileHeader.Size
}
if totalSize > maxSize {
writeJSONError(w, http.StatusRequestEntityTooLarge, uploadTooLargeMessage(maxSize))
return
}
}
var uploadResults []map[string]interface{}
var failedUploads []string
@@ -327,6 +349,15 @@ func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request)
}
}
// uploadFormOverhead allows for the multipart headers and the path field on
// top of the file bytes an upload may carry. Go's multipart parser accepts at
// most 1000 parts, whose headers take about half of this.
const uploadFormOverhead = 1 << 20
func uploadTooLargeMessage(maxSize int64) string {
return fmt.Sprintf("Total upload size exceeds the %d MB limit", maxSize>>20)
}
// uploadFileToFiler uploads a file to the cluster via filer gRPC + volume
// HTTP. This works whether or not the filer is running with -disableHttp=true,
// since the bytes never traverse the filer's HTTP listener. The multipart
@@ -1,8 +1,12 @@
package handlers
import (
"bytes"
"fmt"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"slices"
"strings"
"testing"
@@ -143,3 +147,72 @@ func TestListFolders_RejectsPathOutsideBuckets(t *testing.T) {
}
}
}
func TestUploadFile_RejectsOversizedUpload(t *testing.T) {
adminServer := &dash.AdminServer{}
adminServer.SetUploadMaxSize(1 << 20)
h := &FileBrowserHandlers{adminServer: adminServer}
// 1.5MB and two 600KB files fit the body allowance but exceed the file total;
// 3MB trips the body cap.
for _, sizes := range [][]int{{3 << 19}, {600 << 10, 600 << 10}, {3 << 20}} {
var body bytes.Buffer
mw := multipart.NewWriter(&body)
for i, size := range sizes {
part, err := mw.CreateFormFile("files", fmt.Sprintf("big%d.bin", i))
if err != nil {
t.Fatal(err)
}
part.Write(make([]byte, size))
}
mw.WriteField("path", "/buckets/b")
mw.Close()
req := httptest.NewRequest(http.MethodPost, "/api/files/upload", &body)
req.Header.Set("Content-Type", mw.FormDataContentType())
w := httptest.NewRecorder()
h.UploadFile(w, req)
if w.Code != http.StatusRequestEntityTooLarge {
t.Errorf("sizes %v: expected 413, got %d (body: %s)", sizes, w.Code, w.Body.String())
}
if !strings.Contains(w.Body.String(), "1 MB limit") {
t.Errorf("sizes %v: expected the limit in the error, got %s", sizes, w.Body.String())
}
}
}
func TestUploadFile_RemovesSpooledParts(t *testing.T) {
tmpDir := t.TempDir()
t.Setenv("TMPDIR", tmpDir)
adminServer := &dash.AdminServer{}
adminServer.SetUploadMaxSize(34 << 20)
h := &FileBrowserHandlers{adminServer: adminServer}
// The 33MB part spills to disk; together the parts exceed the limit, so the upload stops there.
var body bytes.Buffer
mw := multipart.NewWriter(&body)
for i, size := range []int{33 << 20, 3 << 19} {
part, err := mw.CreateFormFile("files", fmt.Sprintf("part%d.bin", i))
if err != nil {
t.Fatal(err)
}
part.Write(make([]byte, size))
}
mw.Close()
req := httptest.NewRequest(http.MethodPost, "/api/files/upload", &body)
req.Header.Set("Content-Type", mw.FormDataContentType())
w := httptest.NewRecorder()
h.UploadFile(w, req.WithContext(req.Context()))
if w.Code != http.StatusRequestEntityTooLarge {
t.Fatalf("expected 413, got %d (body: %s)", w.Code, w.Body.String())
}
if left, _ := os.ReadDir(tmpDir); len(left) != 0 {
t.Errorf("spooled parts left behind: %v", left)
}
}
+3 -4
View File
@@ -1180,10 +1180,9 @@ async function submitUploadFile() {
const files = Array.from(fileInput.files);
const totalSize = files.reduce((sum, file) => sum + file.size, 0);
// Validate total file size (limit to 500MB for admin interface)
const maxSize = 500 * 1024 * 1024; // 500MB total
if (totalSize > maxSize) {
showErrorMessage('Total file size exceeds 500MB limit. Please select fewer or smaller files.');
const maxSize = Number(document.getElementById('uploadFileForm').dataset.maxSize) || 0;
if (maxSize > 0 && totalSize > maxSize) {
showErrorMessage(`Total file size exceeds the ${formatBytes(maxSize)} limit. Please select fewer or smaller files.`);
return;
}
Binary file not shown.
+4 -1
View File
@@ -310,12 +310,15 @@ templ FileBrowser(data dash.FileBrowserData) {
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
<form id="uploadFileForm" enctype="multipart/form-data">
<form id="uploadFileForm" enctype="multipart/form-data" data-max-size={ fmt.Sprint(data.UploadMaxSize) }>
<div class="mb-3">
<label for="fileInput" class="form-label">Select Files</label>
<input type="file" class="form-control" id="fileInput" name="files" multiple required>
<div class="form-text">
Choose one or more files to upload to the current directory. You can select multiple files by holding Ctrl (Cmd on Mac) while clicking.
if data.UploadMaxSize > 0 {
Total upload size limit: { formatBytes(data.UploadMaxSize) }.
}
</div>
</div>
<input type="hidden" id="uploadPath" name="path" value={ data.CurrentPath }>
+40 -4
View File
@@ -852,20 +852,56 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 93, "\"></form></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-primary\" onclick=\"submitCreateFolder()\"><i class=\"bi bi-folder-plus me-1\"></i>Create Folder</button></div></div></div></div><!-- Upload File Modal --><div class=\"modal fade\" id=\"uploadFileModal\" tabindex=\"-1\" aria-labelledby=\"uploadFileModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog modal-lg\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"uploadFileModalLabel\"><i class=\"bi bi-upload me-2\"></i>Upload Files</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><form id=\"uploadFileForm\" enctype=\"multipart/form-data\"><div class=\"mb-3\"><label for=\"fileInput\" class=\"form-label\">Select Files</label> <input type=\"file\" class=\"form-control\" id=\"fileInput\" name=\"files\" multiple required><div class=\"form-text\">Choose one or more files to upload to the current directory. You can select multiple files by holding Ctrl (Cmd on Mac) while clicking.</div></div><input type=\"hidden\" id=\"uploadPath\" name=\"path\" value=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 93, "\"></form></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-primary\" onclick=\"submitCreateFolder()\"><i class=\"bi bi-folder-plus me-1\"></i>Create Folder</button></div></div></div></div><!-- Upload File Modal --><div class=\"modal fade\" id=\"uploadFileModal\" tabindex=\"-1\" aria-labelledby=\"uploadFileModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog modal-lg\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"uploadFileModalLabel\"><i class=\"bi bi-upload me-2\"></i>Upload Files</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><form id=\"uploadFileForm\" enctype=\"multipart/form-data\" data-max-size=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var40 string
templ_7745c5c3_Var40, templ_7745c5c3_Err = templruntime.NewAttributeExpression(data.CurrentPath).Resolve(ctx)
templ_7745c5c3_Var40, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprint(data.UploadMaxSize)).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 321, Col: 79}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 313, Col: 107}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var40)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, "\"><!-- File List Preview --><div id=\"fileListPreview\" class=\"mb-3\" style=\"display: none;\"><label class=\"form-label\">Selected Files:</label><div id=\"selectedFilesList\" class=\"border rounded p-2 bg-light\"><!-- Files will be listed here --></div></div><!-- Upload Progress --><div class=\"mb-3\" id=\"uploadProgress\" style=\"display: none;\"><label class=\"form-label\">Upload Progress:</label><div class=\"progress mb-2\"><div class=\"progress-bar progress-bar-striped progress-bar-animated\" role=\"progressbar\" style=\"width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\">0%</div></div><div id=\"uploadStatus\" class=\"small text-muted\">Preparing upload...</div></div></form></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-primary\" onclick=\"submitUploadFile()\"><i class=\"bi bi-upload me-1\"></i>Upload Files</button></div></div></div></div><!-- JavaScript for file browser functionality --><script>\n\tdocument.addEventListener('DOMContentLoaded', function() {\n\t\t// Format permissions in the main table\n\t\tdocument.querySelectorAll('.permissions-display').forEach(element => {\n\t\t\tconst mode = element.getAttribute('data-mode');\n\t\t\tconst isDirectory = element.getAttribute('data-is-directory') === 'true';\n\t\t\tif (mode) {\n\t\t\t\telement.textContent = formatPermissions(mode, isDirectory);\n\t\t\t}\n\t\t});\n\t\t\n\t\t// Handle file browser action buttons (download, view, properties, export-metadata, delete)\n\t\tdocument.addEventListener('click', function(e) {\n\t\t\tconst button = e.target.closest('[data-action]');\n\t\t\tif (!button) return;\n\t\t\t\n\t\t\tconst action = button.getAttribute('data-action');\n\t\t\tif (action === 'copy-url') {\n\t\t\t\tconst url = button.getAttribute('data-url');\n\t\t\t\tif (url) adminCopyToClipboard(url);\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tconst path = button.getAttribute('data-path');\n\n\t\t\tif (!path) return;\n\t\t\t\n\t\t\tswitch(action) {\n\t\t\t\tcase 'download':\n\t\t\t\t\tdownloadFile(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'view':\n\t\t\t\t\tviewFile(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'properties':\n\t\t\t\t\tshowFileProperties(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'export-metadata':\n\t\t\t\t\texportMetadata(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'delete':\n\t\t\t\t\tconst fileName = path.split('/').pop();\n\t\t\t\t\tshowDeleteConfirm(fileName, function() {\n\t\t\t\t\t\tdeleteFile(path);\n\t\t\t\t\t}, `Are you sure you want to delete \"${fileName}\"? This action cannot be undone.`);\n\t\t\t\t\tbreak;\n\t\t\t}\n\t\t});\n\t\t\n\t\t// Initialize file manager event handlers from admin.js\n\t\tif (typeof setupFileManagerEventHandlers === 'function') {\n\t\t\tsetupFileManagerEventHandlers();\n\t\t}\n\t});\n\t\n\t// File browser specific functions\n\tfunction downloadFile(path) {\n\t\t// Open download URL in new tab\n\t\twindow.open((window.__BASE_PATH__ || '') + '/api/files/download?path=' + encodeURIComponent(path), '_blank');\n\t}\n\n\tfunction viewFile(path) {\n\t\t// Open file viewer in new tab\n\t\twindow.open((window.__BASE_PATH__ || '') + '/api/files/view?path=' + encodeURIComponent(path), '_blank');\n\t}\n\n\tfunction exportMetadata(path) {\n\t\t// Download metadata as an fs.meta.load-compatible .meta.gz\n\t\twindow.open((window.__BASE_PATH__ || '') + '/api/files/metadata?path=' + encodeURIComponent(path), '_blank');\n\t}\n\n\tfunction showFileProperties(path) {\n\t\t// Fetch file properties and show in modal\n\t\tfetch((window.__BASE_PATH__ || '') + '/api/files/properties?path=' + encodeURIComponent(path))\n\t\t\t.then(response => response.json())\n\t\t\t.then(data => {\n\t\t\t\tif (data.error) {\n\t\t\t\t\tshowAlert('Error loading file properties: ' + data.error, 'error');\n\t\t\t\t} else {\n\t\t\t\t\tdisplayFileProperties(data);\n\t\t\t\t}\n\t\t\t})\n\t\t\t.catch(error => {\n\t\t\t\tconsole.error('Error fetching file properties:', error);\n\t\t\t\tshowAlert('Error loading file properties: ' + error.message, 'error');\n\t\t\t});\n\t}\n\t\n\tfunction displayFileProperties(data) {\n\t\t// Create a comprehensive modal for file properties\n\t\tconst modalHtml = '<div class=\"modal fade\" id=\"filePropertiesModal\" tabindex=\"-1\">' +\n\t\t\t'<div class=\"modal-dialog modal-lg\">' +\n\t\t\t'<div class=\"modal-content\">' +\n\t\t\t'<div class=\"modal-header\">' +\n\t\t\t'<h5 class=\"modal-title\"><i class=\"bi bi-info-circle me-2\"></i>Properties: ' + (data.name || 'Unknown') + '</h5>' +\n\t\t\t'<button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\"></button>' +\n\t\t\t'</div>' +\n\t\t\t'<div class=\"modal-body\">' +\n\t\t\tcreateFilePropertiesContent(data) +\n\t\t\t'</div>' +\n\t\t\t'<div class=\"modal-footer\">' +\n\t\t\t'<button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Close</button>' Line truncated
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, "\"><div class=\"mb-3\"><label for=\"fileInput\" class=\"form-label\">Select Files</label> <input type=\"file\" class=\"form-control\" id=\"fileInput\" name=\"files\" multiple required><div class=\"form-text\">Choose one or more files to upload to the current directory. You can select multiple files by holding Ctrl (Cmd on Mac) while clicking. ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if data.UploadMaxSize > 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 95, "Total upload size limit: ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var41 string
templ_7745c5c3_Var41, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(data.UploadMaxSize))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 320, Col: 67}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var41))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, ".")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 97, "</div></div><input type=\"hidden\" id=\"uploadPath\" name=\"path\" value=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var42 string
templ_7745c5c3_Var42, templ_7745c5c3_Err = templruntime.NewAttributeExpression(data.CurrentPath).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 324, Col: 79}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var42)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 98, "\"><!-- File List Preview --><div id=\"fileListPreview\" class=\"mb-3\" style=\"display: none;\"><label class=\"form-label\">Selected Files:</label><div id=\"selectedFilesList\" class=\"border rounded p-2 bg-light\"><!-- Files will be listed here --></div></div><!-- Upload Progress --><div class=\"mb-3\" id=\"uploadProgress\" style=\"display: none;\"><label class=\"form-label\">Upload Progress:</label><div class=\"progress mb-2\"><div class=\"progress-bar progress-bar-striped progress-bar-animated\" role=\"progressbar\" style=\"width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\">0%</div></div><div id=\"uploadStatus\" class=\"small text-muted\">Preparing upload...</div></div></form></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-primary\" onclick=\"submitUploadFile()\"><i class=\"bi bi-upload me-1\"></i>Upload Files</button></div></div></div></div><!-- JavaScript for file browser functionality --><script>\n\tdocument.addEventListener('DOMContentLoaded', function() {\n\t\t// Format permissions in the main table\n\t\tdocument.querySelectorAll('.permissions-display').forEach(element => {\n\t\t\tconst mode = element.getAttribute('data-mode');\n\t\t\tconst isDirectory = element.getAttribute('data-is-directory') === 'true';\n\t\t\tif (mode) {\n\t\t\t\telement.textContent = formatPermissions(mode, isDirectory);\n\t\t\t}\n\t\t});\n\t\t\n\t\t// Handle file browser action buttons (download, view, properties, export-metadata, delete)\n\t\tdocument.addEventListener('click', function(e) {\n\t\t\tconst button = e.target.closest('[data-action]');\n\t\t\tif (!button) return;\n\t\t\t\n\t\t\tconst action = button.getAttribute('data-action');\n\t\t\tif (action === 'copy-url') {\n\t\t\t\tconst url = button.getAttribute('data-url');\n\t\t\t\tif (url) adminCopyToClipboard(url);\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tconst path = button.getAttribute('data-path');\n\n\t\t\tif (!path) return;\n\t\t\t\n\t\t\tswitch(action) {\n\t\t\t\tcase 'download':\n\t\t\t\t\tdownloadFile(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'view':\n\t\t\t\t\tviewFile(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'properties':\n\t\t\t\t\tshowFileProperties(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'export-metadata':\n\t\t\t\t\texportMetadata(path);\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'delete':\n\t\t\t\t\tconst fileName = path.split('/').pop();\n\t\t\t\t\tshowDeleteConfirm(fileName, function() {\n\t\t\t\t\t\tdeleteFile(path);\n\t\t\t\t\t}, `Are you sure you want to delete \"${fileName}\"? This action cannot be undone.`);\n\t\t\t\t\tbreak;\n\t\t\t}\n\t\t});\n\t\t\n\t\t// Initialize file manager event handlers from admin.js\n\t\tif (typeof setupFileManagerEventHandlers === 'function') {\n\t\t\tsetupFileManagerEventHandlers();\n\t\t}\n\t});\n\t\n\t// File browser specific functions\n\tfunction downloadFile(path) {\n\t\t// Open download URL in new tab\n\t\twindow.open((window.__BASE_PATH__ || '') + '/api/files/download?path=' + encodeURIComponent(path), '_blank');\n\t}\n\n\tfunction viewFile(path) {\n\t\t// Open file viewer in new tab\n\t\twindow.open((window.__BASE_PATH__ || '') + '/api/files/view?path=' + encodeURIComponent(path), '_blank');\n\t}\n\n\tfunction exportMetadata(path) {\n\t\t// Download metadata as an fs.meta.load-compatible .meta.gz\n\t\twindow.open((window.__BASE_PATH__ || '') + '/api/files/metadata?path=' + encodeURIComponent(path), '_blank');\n\t}\n\n\tfunction showFileProperties(path) {\n\t\t// Fetch file properties and show in modal\n\t\tfetch((window.__BASE_PATH__ || '') + '/api/files/properties?path=' + encodeURIComponent(path))\n\t\t\t.then(response => response.json())\n\t\t\t.then(data => {\n\t\t\t\tif (data.error) {\n\t\t\t\t\tshowAlert('Error loading file properties: ' + data.error, 'error');\n\t\t\t\t} else {\n\t\t\t\t\tdisplayFileProperties(data);\n\t\t\t\t}\n\t\t\t})\n\t\t\t.catch(error => {\n\t\t\t\tconsole.error('Error fetching file properties:', error);\n\t\t\t\tshowAlert('Error loading file properties: ' + error.message, 'error');\n\t\t\t});\n\t}\n\t\n\tfunction displayFileProperties(data) {\n\t\t// Create a comprehensive modal for file properties\n\t\tconst modalHtml = '<div class=\"modal fade\" id=\"filePropertiesModal\" tabindex=\"-1\">' +\n\t\t\t'<div class=\"modal-dialog modal-lg\">' +\n\t\t\t'<div class=\"modal-content\">' +\n\t\t\t'<div class=\"modal-header\">' +\n\t\t\t'<h5 class=\"modal-title\"><i class=\"bi bi-info-circle me-2\"></i>Properties: ' + (data.name || 'Unknown') + '</h5>' +\n\t\t\t'<button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\"></button>' +\n\t\t\t'</div>' +\n\t\t\t'<div class=\"modal-body\">' +\n\t\t\tcreateFilePropertiesContent(data) +\n\t\t\t'</div>' +\n\t\t\t'<div class=\"modal-footer\">' +\n\t\t\t'<button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Close</button>' Line truncated
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -6,6 +6,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
)
templ S3TablesBuckets(data dash.S3TablesBucketsData) {
@@ -160,6 +161,9 @@ templ S3TablesBuckets(data dash.S3TablesBucketsData) {
<td>{ bucket.Name }</td>
<td>
<span class={ formatBadgeClass(bucket.Format) } title={ formatBadgeTitle(bucket.Format) }>{ formatLabel(bucket.Format) }</span>
if s3vectors.IsVectorBucketFormat(bucket.Format) {
<a class="ms-1 small" href={ dash.PUrl(ctx, "/object-store/vector-buckets/"+url.PathEscape(bucket.Name)) } title="This bucket's S3 Vectors indexes and policy">vector bucket</a>
}
</td>
<td>{ bucket.OwnerAccountID }</td>
<td class="text-muted small">{ bucket.ARN }</td>
+157 -133
View File
@@ -15,6 +15,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
)
func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
@@ -46,7 +47,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
var templ_7745c5c3_Var2 string
templ_7745c5c3_Var2, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.SafeURL(fmt.Sprintf("//localhost:%d/v1/config", data.IcebergPort))).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 24, Col: 119}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 25, Col: 119}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var2)
if templ_7745c5c3_Err != nil {
@@ -65,7 +66,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
var templ_7745c5c3_Var3 string
templ_7745c5c3_Var3, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.SafeURL(fmt.Sprintf("//localhost:%d/v1/table", data.LancePort))).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 31, Col: 114}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 32, Col: 114}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var3)
if templ_7745c5c3_Err != nil {
@@ -83,7 +84,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
var templ_7745c5c3_Var4 string
templ_7745c5c3_Var4, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.IcebergPort)).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 38, Col: 91}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 39, Col: 91}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var4)
if templ_7745c5c3_Err != nil {
@@ -96,7 +97,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
var templ_7745c5c3_Var5 string
templ_7745c5c3_Var5, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.LancePort)).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 38, Col: 145}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 39, Col: 145}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var5)
if templ_7745c5c3_Err != nil {
@@ -119,7 +120,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
var templ_7745c5c3_Var6 string
templ_7745c5c3_Var6, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.IcebergPort)).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 52, Col: 105}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 53, Col: 105}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var6)
if templ_7745c5c3_Err != nil {
@@ -132,7 +133,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
var templ_7745c5c3_Var7 string
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("http://localhost:%d", data.IcebergPort))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 52, Col: 162}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 53, Col: 162}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
if templ_7745c5c3_Err != nil {
@@ -151,7 +152,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
var templ_7745c5c3_Var8 string
templ_7745c5c3_Var8, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.LancePort)).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 58, Col: 103}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 59, Col: 103}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var8)
if templ_7745c5c3_Err != nil {
@@ -164,7 +165,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
var templ_7745c5c3_Var9 string
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("http://localhost:%d", data.LancePort))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 58, Col: 158}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 59, Col: 158}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
if templ_7745c5c3_Err != nil {
@@ -193,7 +194,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
var templ_7745c5c3_Var10 string
templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalBuckets))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 81, Col: 47}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 82, Col: 47}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
if templ_7745c5c3_Err != nil {
@@ -206,7 +207,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
var templ_7745c5c3_Var11 string
templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04"))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 100, Col: 54}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 101, Col: 54}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
if templ_7745c5c3_Err != nil {
@@ -220,7 +221,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
var templ_7745c5c3_Var12 string
templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 120, Col: 47}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 121, Col: 47}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
if templ_7745c5c3_Err != nil {
@@ -244,7 +245,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
var templ_7745c5c3_Var13 string
templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 160, Col: 28}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 161, Col: 28}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
if templ_7745c5c3_Err != nil {
@@ -283,7 +284,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
var templ_7745c5c3_Var16 string
templ_7745c5c3_Var16, templ_7745c5c3_Err = templruntime.NewAttributeExpression(formatBadgeTitle(bucket.Format)).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 162, Col: 99}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 163, Col: 99}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var16)
if templ_7745c5c3_Err != nil {
@@ -310,220 +311,243 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
var templ_7745c5c3_Var17 string
templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(formatLabel(bucket.Format))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 162, Col: 130}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 163, Col: 130}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "</span></td><td>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "</span> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var18 string
templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.OwnerAccountID)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 164, Col: 38}
if s3vectors.IsVectorBucketFormat(bucket.Format) {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "<a class=\"ms-1 small\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var18 string
templ_7745c5c3_Var18, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, "/object-store/vector-buckets/"+url.PathEscape(bucket.Name))).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 165, Col: 117}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var18)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "\" title=\"This bucket's S3 Vectors indexes and policy\">vector bucket</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "</td><td class=\"text-muted small\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "</td><td>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var19 string
templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.OwnerAccountID)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 165, Col: 52}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 168, Col: 38}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "</td><td><code class=\"small\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "</td><td class=\"text-muted small\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var20 string
templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(bucketCatalogPath(bucket.Format, bucket.Name))
templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 166, Col: 82}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 169, Col: 52}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "</code></td><td>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "</td><td><code class=\"small\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var21 string
templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04"))
templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(bucketCatalogPath(bucket.Format, bucket.Name))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 167, Col: 60}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 170, Col: 82}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "</td><td>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "</code></td><td>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var22 string
templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04"))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 171, Col: 60}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "</td><td>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if bucket.PolicyStatementCount > 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "<button type=\"button\" class=\"badge bg-primary border-0 s3tables-bucket-policy-btn\" data-bucket-arn=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "<button type=\"button\" class=\"badge bg-primary border-0 s3tables-bucket-policy-btn\" data-bucket-arn=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var22 string
templ_7745c5c3_Var22, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.ARN).Resolve(ctx)
var templ_7745c5c3_Var23 string
templ_7745c5c3_Var23, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.ARN).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 171, Col: 43}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 175, Col: 43}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var22)
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var23)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "\" title=\"View Bucket Policy\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "\" title=\"View Bucket Policy\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if bucket.PolicyStatementCount == 1 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "1 statement")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "1 statement")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
var templ_7745c5c3_Var23 string
templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d statements", bucket.PolicyStatementCount))
var templ_7745c5c3_Var24 string
templ_7745c5c3_Var24, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d statements", bucket.PolicyStatementCount))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 175, Col: 74}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 179, Col: 74}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var24))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "</button>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "</button>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "<span class=\"text-muted\">Not configured</span>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "<span class=\"text-muted\">Not configured</span>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "</td><td><div class=\"btn-group btn-group-sm\" role=\"group\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "</td><td><div class=\"btn-group btn-group-sm\" role=\"group\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
bucketName, parseErr := s3tables.ParseBucketNameFromARN(bucket.ARN)
if parseErr == nil {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "<a class=\"btn btn-outline-primary btn-sm\" href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "<a class=\"btn btn-outline-primary btn-sm\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var24 string
templ_7745c5c3_Var24, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName)))).ResolveURL(ctx)
var templ_7745c5c3_Var25 string
templ_7745c5c3_Var25, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName)))).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 186, Col: 166}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 190, Col: 166}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var24)
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var25)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "\"><i class=\"bi bi-folder2-open\"></i></a> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "\"><i class=\"bi bi-folder2-open\"></i></a> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "<button type=\"button\" class=\"btn btn-outline-primary btn-sm\" disabled title=\"Invalid bucket ARN\"><i class=\"bi bi-folder2-open\"></i></button> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "<button type=\"button\" class=\"btn btn-outline-primary btn-sm\" disabled title=\"Invalid bucket ARN\"><i class=\"bi bi-folder2-open\"></i></button> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "<button type=\"button\" class=\"btn btn-outline-success btn-sm s3tables-tags-btn\" data-resource-arn=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var25 string
templ_7745c5c3_Var25, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.ARN).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 194, Col: 122}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var25)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "\" title=\"Tags\"><i class=\"bi bi-tags\"></i></button> <button type=\"button\" class=\"btn btn-outline-info btn-sm s3tables-bucket-policy-btn\" data-bucket-arn=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "<button type=\"button\" class=\"btn btn-outline-success btn-sm s3tables-tags-btn\" data-resource-arn=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var26 string
templ_7745c5c3_Var26, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.ARN).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 197, Col: 126}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 198, Col: 122}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var26)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "\" title=\"Bucket Policy\"><i class=\"bi bi-shield\"></i></button> <button type=\"button\" class=\"btn btn-outline-danger btn-sm s3tables-delete-bucket-btn\" data-bucket-arn=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "\" title=\"Tags\"><i class=\"bi bi-tags\"></i></button> <button type=\"button\" class=\"btn btn-outline-info btn-sm s3tables-bucket-policy-btn\" data-bucket-arn=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var27 string
templ_7745c5c3_Var27, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.ARN).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 200, Col: 128}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 201, Col: 126}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var27)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "\" data-bucket-name=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "\" title=\"Bucket Policy\"><i class=\"bi bi-shield\"></i></button> <button type=\"button\" class=\"btn btn-outline-danger btn-sm s3tables-delete-bucket-btn\" data-bucket-arn=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var28 string
templ_7745c5c3_Var28, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.Name).Resolve(ctx)
templ_7745c5c3_Var28, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.ARN).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 200, Col: 161}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 204, Col: 128}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var28)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "\" title=\"Delete\"><i class=\"bi bi-trash\"></i></button></div></td></tr>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "\" data-bucket-name=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var29 string
templ_7745c5c3_Var29, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.Name).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 204, Col: 161}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var29)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "\" title=\"Delete\"><i class=\"bi bi-trash\"></i></button></div></td></tr>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
if len(data.Buckets) == 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "<tr><td colspan=\"8\" class=\"text-center text-muted py-4\"><i class=\"bi bi-table icon-3x mb-3 text-muted\"></i><div><h5>No table buckets found</h5><p>Create your first S3 Tables bucket to get started.</p><button type=\"button\" class=\"btn btn-primary\" data-bs-toggle=\"modal\" data-bs-target=\"#createS3TablesBucketModal\"><i class=\"bi bi-plus me-1\"></i>Create Bucket</button></div></td></tr>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "<tr><td colspan=\"8\" class=\"text-center text-muted py-4\"><i class=\"bi bi-table icon-3x mb-3 text-muted\"></i><div><h5>No table buckets found</h5><p>Create your first S3 Tables bucket to get started.</p><button type=\"button\" class=\"btn btn-primary\" data-bs-toggle=\"modal\" data-bs-target=\"#createS3TablesBucketModal\"><i class=\"bi bi-plus me-1\"></i>Create Bucket</button></div></td></tr>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "</tbody></table></div></div></div></div></div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "</tbody></table></div></div></div></div></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if data.IcebergPort > 0 || data.LancePort > 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "<div class=\"row\"><div class=\"col-12\"><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-code me-2\"></i>Client Examples</h6></div><div class=\"card-body\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "<div class=\"row\"><div class=\"col-12\"><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-code me-2\"></i>Client Examples</h6></div><div class=\"card-body\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if data.IcebergPort > 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "<h6><span class=\"badge bg-primary me-2\">ICEBERG</span>DuckDB</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-duckdb-example\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "<h6><span class=\"badge bg-primary me-2\">ICEBERG</span>DuckDB</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-duckdb-example\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var29 string
templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(`INSTALL iceberg;
var templ_7745c5c3_Var30 string
templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(`INSTALL iceberg;
LOAD iceberg;
CREATE SECRET (
@@ -534,18 +558,18 @@ CREATE SECRET (
SELECT * FROM iceberg_scan('s3://my-table-bucket/my-namespace/my-table');`)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 251, Col: 74}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 255, Col: 74}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "</code></pre><h6 class=\"mt-4\"><span class=\"badge bg-primary me-2\">ICEBERG</span>Python (PyIceberg)</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-pyiceberg-example\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "</code></pre><h6 class=\"mt-4\"><span class=\"badge bg-primary me-2\">ICEBERG</span>Python (PyIceberg)</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-pyiceberg-example\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var30 string
templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(`from pyiceberg.catalog import load_catalog
var templ_7745c5c3_Var31 string
templ_7745c5c3_Var31, templ_7745c5c3_Err = templ.JoinStringErrs(`from pyiceberg.catalog import load_catalog
catalog = load_catalog(
name="seaweedfs",
@@ -558,13 +582,13 @@ catalog = load_catalog(
namespaces = catalog.list_namespaces()`)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 268, Col: 39}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 272, Col: 39}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var31))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "</code></pre>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "</code></pre>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -574,25 +598,25 @@ namespaces = catalog.list_namespaces()`)
templ_7745c5c3_ElementWriter := templ_7745c5c3_Buffer
templ_7745c5c3_Buffer := templ.GetBuffer()
var templ_7745c5c3_ElementScripts []templ.ComponentScript
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "<h6 class=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "<h6 class=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var31 = []any{templ.KV("mt-4", data.IcebergPort > 0)}
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_ElementWriter, templ_7745c5c3_Var31...)
var templ_7745c5c3_Var32 = []any{templ.KV("mt-4", data.IcebergPort > 0)}
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_ElementWriter, templ_7745c5c3_Var32...)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var32 string
templ_7745c5c3_Var32, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.CSSClasses(templ_7745c5c3_Var31).String()).Resolve(ctx)
var templ_7745c5c3_Var33 string
templ_7745c5c3_Var33, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.CSSClasses(templ_7745c5c3_Var32).String()).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 1, Col: 0}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var32)
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var33)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -606,30 +630,30 @@ namespaces = catalog.list_namespaces()`)
}
templ.ReleaseBuffer(templ_7745c5c3_Buffer)
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "><span class=\"badge bg-success me-2\">LANCE</span>Python (lance-namespace)</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-lance-namespace-example\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "><span class=\"badge bg-success me-2\">LANCE</span>Python (lance-namespace)</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-lance-namespace-example\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var33 string
templ_7745c5c3_Var33, templ_7745c5c3_Err = templ.JoinStringErrs(`from lance_namespace import connect
var templ_7745c5c3_Var34 string
templ_7745c5c3_Var34, templ_7745c5c3_Err = templ.JoinStringErrs(`from lance_namespace import connect
ns = connect("rest", {"uri": "http://localhost:` + fmt.Sprintf("%d", data.LancePort) + `"})
ns.list_namespaces(id=["my-table-bucket"])
ns.describe_table(id=["my-table-bucket", "my-namespace", "my-table"])`)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 281, Col: 70}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 285, Col: 70}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var33))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var34))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "</code></pre><h6 class=\"mt-4\"><span class=\"badge bg-success me-2\">LANCE</span>Python (pylance)</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-pylance-example\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "</code></pre><h6 class=\"mt-4\"><span class=\"badge bg-success me-2\">LANCE</span>Python (pylance)</h6><pre class=\"bg-light p-3 border rounded\"><code id=\"s3tables-pylance-example\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var34 string
templ_7745c5c3_Var34, templ_7745c5c3_Err = templ.JoinStringErrs(`import lance
var templ_7745c5c3_Var35 string
templ_7745c5c3_Var35, templ_7745c5c3_Err = templ.JoinStringErrs(`import lance
from lance_namespace import connect
ns = connect("rest", {"uri": "http://localhost:` + fmt.Sprintf("%d", data.LancePort) + `"})
@@ -638,84 +662,84 @@ table = ns.describe_table(id=["my-table-bucket", "my-namespace", "my-table"])
# The namespace vends both the location and the credentials to read it.
dataset = lance.dataset(table.location, storage_options=table.storage_options)`)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 294, Col: 79}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 298, Col: 79}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var34))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var35))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "</code></pre>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "</code></pre>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "<script>\n\t\t\t\t\t\t\t\tconst s3tablesExamplesHost = window.location.hostname;\n\t\t\t\t\t\t\t\tconst s3tablesExamples = [\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-duckdb-example'),\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-pyiceberg-example'),\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-lance-namespace-example'),\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-pylance-example')\n\t\t\t\t\t\t\t\t];\n\t\t\t\t\t\t\t\ts3tablesExamples.forEach(example => {\n\t\t\t\t\t\t\t\t\tif (!example) {\n\t\t\t\t\t\t\t\t\t\treturn;\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\texample.textContent = example.textContent.replaceAll('localhost', s3tablesExamplesHost);\n\t\t\t\t\t\t\t\t});\n\t\t\t\t\t\t\t</script></div></div></div></div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "<script>\n\t\t\t\t\t\t\t\tconst s3tablesExamplesHost = window.location.hostname;\n\t\t\t\t\t\t\t\tconst s3tablesExamples = [\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-duckdb-example'),\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-pyiceberg-example'),\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-lance-namespace-example'),\n\t\t\t\t\t\t\t\t\tdocument.getElementById('s3tables-pylance-example')\n\t\t\t\t\t\t\t\t];\n\t\t\t\t\t\t\t\ts3tablesExamples.forEach(example => {\n\t\t\t\t\t\t\t\t\tif (!example) {\n\t\t\t\t\t\t\t\t\t\treturn;\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\texample.textContent = example.textContent.replaceAll('localhost', s3tablesExamplesHost);\n\t\t\t\t\t\t\t\t});\n\t\t\t\t\t\t\t</script></div></div></div></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "</div><div class=\"modal fade\" id=\"createS3TablesBucketModal\" tabindex=\"-1\" aria-labelledby=\"createS3TablesBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"createS3TablesBucketModalLabel\"><i class=\"bi bi-plus me-2\"></i>Create Table Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"createS3TablesBucketForm\"><div class=\"modal-body\"><div class=\"mb-3\"><label for=\"s3tablesBucketName\" class=\"form-label\">Bucket Name</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesBucketName\" name=\"name\" placeholder=\"table-bucket-name\" required></div><div class=\"mb-3\"><label class=\"form-label\">Table Format</label><div class=\"row g-2\" id=\"s3tablesBucketFormatPicker\"><div class=\"col-6\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "</div><div class=\"modal fade\" id=\"createS3TablesBucketModal\" tabindex=\"-1\" aria-labelledby=\"createS3TablesBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"createS3TablesBucketModalLabel\"><i class=\"bi bi-plus me-2\"></i>Create Table Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"createS3TablesBucketForm\"><div class=\"modal-body\"><div class=\"mb-3\"><label for=\"s3tablesBucketName\" class=\"form-label\">Bucket Name</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesBucketName\" name=\"name\" placeholder=\"table-bucket-name\" required></div><div class=\"mb-3\"><label class=\"form-label\">Table Format</label><div class=\"row g-2\" id=\"s3tablesBucketFormatPicker\"><div class=\"col-6\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if data.IcebergPort > 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatIceberg\" value=\"ICEBERG\" checked> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatIceberg\" value=\"ICEBERG\" checked> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatIceberg\" value=\"ICEBERG\" disabled> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatIceberg\" value=\"ICEBERG\" disabled> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "<label class=\"btn btn-outline-primary w-100 text-start p-3\" for=\"s3tablesBucketFormatIceberg\"><span class=\"d-block fw-semibold\"><i class=\"bi bi-snow me-1\"></i>Iceberg</span> <span class=\"d-block small text-muted mt-1\">Served over the Iceberg REST catalog. Spark, Trino, PyIceberg, DuckDB, OLake Go.</span></label></div><div class=\"col-6\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, "<label class=\"btn btn-outline-primary w-100 text-start p-3\" for=\"s3tablesBucketFormatIceberg\"><span class=\"d-block fw-semibold\"><i class=\"bi bi-snow me-1\"></i>Iceberg</span> <span class=\"d-block small text-muted mt-1\">Served over the Iceberg REST catalog. Spark, Trino, PyIceberg, DuckDB, OLake Go.</span></label></div><div class=\"col-6\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if data.LancePort > 0 && data.IcebergPort <= 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, " <input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatLance\" value=\"LANCE\" checked> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, " <input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatLance\" value=\"LANCE\" checked> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else if data.LancePort > 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatLance\" value=\"LANCE\"> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 68, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatLance\" value=\"LANCE\"> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatLance\" value=\"LANCE\" disabled> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 69, "<input type=\"radio\" class=\"btn-check\" name=\"format\" id=\"s3tablesBucketFormatLance\" value=\"LANCE\" disabled> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, "<label class=\"btn btn-outline-success w-100 text-start p-3\" for=\"s3tablesBucketFormatLance\"><span class=\"d-block fw-semibold\"><i class=\"bi bi-bounding-box me-1\"></i>Lance</span> <span class=\"d-block small text-muted mt-1\">Served over the Lance Namespace API. pylance, LanceDB, Ray.</span></label></div></div><div class=\"form-text\" id=\"s3tablesBucketFormatHint\" data-iceberg-port=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var35 string
templ_7745c5c3_Var35, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.IcebergPort)).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 363, Col: 115}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var35)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 68, "\" data-lance-port=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 70, "<label class=\"btn btn-outline-success w-100 text-start p-3\" for=\"s3tablesBucketFormatLance\"><span class=\"d-block fw-semibold\"><i class=\"bi bi-bounding-box me-1\"></i>Lance</span> <span class=\"d-block small text-muted mt-1\">Served over the Lance Namespace API. pylance, LanceDB, Ray.</span></label></div></div><div class=\"form-text\" id=\"s3tablesBucketFormatHint\" data-iceberg-port=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var36 string
templ_7745c5c3_Var36, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.LancePort)).Resolve(ctx)
templ_7745c5c3_Var36, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.IcebergPort)).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 363, Col: 169}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 367, Col: 115}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var36)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 69, "\">A bucket holds one format. Tables of the other are refused.</div></div><div class=\"mb-3\"><label for=\"s3tablesBucketOwner\" class=\"form-label\">Owner (Optional)</label> <select class=\"form-select\" id=\"s3tablesBucketOwner\" name=\"owner\"><option value=\"\">No owner (admin-only access)</option></select><div class=\"form-text\">The S3 identity that owns this table bucket. Non-admin users can only access table buckets they own.</div></div><div class=\"mb-3\"><label for=\"s3tablesBucketTags\" class=\"form-label\">Tags</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesBucketTags\" name=\"tags\" placeholder=\"key1=value1,key2=value2\"><div class=\"form-text\">Optional tags in key=value format.</div></div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"submit\" class=\"btn btn-primary\"><i class=\"bi bi-plus me-1\"></i>Create</button></div></form></div></div></div><div class=\"modal fade\" id=\"deleteS3TablesBucketModal\" tabindex=\"-1\" aria-labelledby=\"deleteS3TablesBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"deleteS3TablesBucketModalLabel\"><i class=\"bi bi-exclamation-triangle me-2 text-warning\"></i>Delete Table Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p>Are you sure you want to delete the table bucket <strong id=\"deleteS3TablesBucketName\"></strong>?</p></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-danger\" onclick=\"deleteS3TablesBucket()\"><i class=\"bi bi-trash me-1\"></i>Delete</button></div></div></div></div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 71, "\" data-lance-port=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var37 string
templ_7745c5c3_Var37, templ_7745c5c3_Err = templruntime.NewAttributeExpression(fmt.Sprintf("%d", data.LancePort)).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 367, Col: 169}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var37)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 72, "\">A bucket holds one format. Tables of the other are refused.</div></div><div class=\"mb-3\"><label for=\"s3tablesBucketOwner\" class=\"form-label\">Owner (Optional)</label> <select class=\"form-select\" id=\"s3tablesBucketOwner\" name=\"owner\"><option value=\"\">No owner (admin-only access)</option></select><div class=\"form-text\">The S3 identity that owns this table bucket. Non-admin users can only access table buckets they own.</div></div><div class=\"mb-3\"><label for=\"s3tablesBucketTags\" class=\"form-label\">Tags</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesBucketTags\" name=\"tags\" placeholder=\"key1=value1,key2=value2\"><div class=\"form-text\">Optional tags in key=value format.</div></div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"submit\" class=\"btn btn-primary\"><i class=\"bi bi-plus me-1\"></i>Create</button></div></form></div></div></div><div class=\"modal fade\" id=\"deleteS3TablesBucketModal\" tabindex=\"-1\" aria-labelledby=\"deleteS3TablesBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"deleteS3TablesBucketModalLabel\"><i class=\"bi bi-exclamation-triangle me-2 text-warning\"></i>Delete Table Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p>Are you sure you want to delete the table bucket <strong id=\"deleteS3TablesBucketName\"></strong>?</p></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-danger\" onclick=\"deleteS3TablesBucket()\"><i class=\"bi bi-trash me-1\"></i>Delete</button></div></div></div></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -723,7 +747,7 @@ dataset = lance.dataset(table.location, storage_options=table.storage_options)`)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 70, "<div class=\"modal fade\" id=\"s3tablesBucketPolicyModal\" tabindex=\"-1\" aria-labelledby=\"s3tablesBucketPolicyModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog modal-xl\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"s3tablesBucketPolicyModalLabel\"><i class=\"bi bi-shield me-2\"></i>Table Bucket Policy</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"s3tablesBucketPolicyForm\"><div class=\"modal-body\"><input type=\"hidden\" id=\"s3tablesBucketPolicyArn\" name=\"bucket_arn\"><ul class=\"nav nav-tabs\" role=\"tablist\"><li class=\"nav-item\" role=\"presentation\"><button class=\"nav-link active\" id=\"s3tablesBucketPolicyEditorTabBtn\" type=\"button\" data-bs-toggle=\"tab\" data-bs-target=\"#s3tablesBucketPolicyEditorTab\" role=\"tab\" aria-controls=\"s3tablesBucketPolicyEditorTab\" aria-selected=\"true\">Editor</button></li><li class=\"nav-item\" role=\"presentation\"><button class=\"nav-link\" id=\"s3tablesBucketPolicyJsonTabBtn\" type=\"button\" data-bs-toggle=\"tab\" data-bs-target=\"#s3tablesBucketPolicyJsonTab\" role=\"tab\" aria-controls=\"s3tablesBucketPolicyJsonTab\" aria-selected=\"false\">JSON</button></li></ul><div class=\"tab-content border border-top-0 rounded-bottom p-3 mb-3\"><div class=\"tab-pane fade show active\" id=\"s3tablesBucketPolicyEditorTab\" role=\"tabpanel\" aria-labelledby=\"s3tablesBucketPolicyEditorTabBtn\"><div id=\"s3tablesBucketPolicyEditorBody\"></div><button type=\"button\" class=\"btn btn-sm btn-outline-secondary\" id=\"s3tablesBucketPolicyAddStatementBtn\"><i class=\"bi bi-plus me-1\"></i>Add statement</button></div><div class=\"tab-pane fade\" id=\"s3tablesBucketPolicyJsonTab\" role=\"tabpanel\" aria-labelledby=\"s3tablesBucketPolicyJsonTabBtn\"><textarea class=\"form-control\" id=\"s3tablesBucketPolicyText\" name=\"policy\" rows=\"12\" placeholder=\"{ }\"></textarea></div></div><div class=\"form-text\">Provide a policy JSON; use Delete Policy to remove the policy.</div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Close</button> <button type=\"button\" class=\"btn btn-outline-danger\" id=\"s3tablesBucketPolicyDeleteBtn\" onclick=\"deleteS3TablesBucketPolicy()\"><i class=\"bi bi-trash me-1\"></i>Delete Policy</button><!-- type=\"button\": a submit button would make Enter in any\n\t\t\t\t\t\t single-line editor input implicitly submit a half-built\n\t\t\t\t\t\t policy, which the backend stores verbatim. --><button type=\"button\" class=\"btn btn-primary\" id=\"s3tablesBucketPolicySaveBtn\"><i class=\"bi bi-floppy me-1\"></i>Save Policy</button></div></form></div></div></div><div class=\"modal fade\" id=\"s3tablesTagsModal\" tabindex=\"-1\" aria-labelledby=\"s3tablesTagsModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog modal-lg\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"s3tablesTagsModalLabel\"><i class=\"bi bi-tags me-2\"></i>Resource Tags</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"s3tablesTagsForm\"><div class=\"modal-body\"><input type=\"hidden\" id=\"s3tablesTagsResourceArn\" name=\"resource_arn\"><div class=\"mb-3\"><label class=\"form-label\">Existing Tags</label><pre class=\"bg-light p-3 border rounded\" id=\"s3tablesTagsList\">Loading...</pre></div><div class=\"mb-3\"><label for=\"s3tablesTagsInput\" class=\"form-label\">Add or Update Tags</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesTagsInput\" placeholder=\"key1=value1,key2=value2\"></div><div class=\"mb-3\"><label for=\"s3tablesTagsDeleteInput\" class=\"form-label\">Remove Tag Keys</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesTagsDeleteInput\" placeholder=\"key1,key2\"></div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Close</button> <button type=\"button\" class=\"btn btn-outline-danger\" onclick=\"deleteS3TablesTags()\"><i class=\"bi bi-trash me-1\"></i>Remove Tags</button> <button type=\"submit\" class=\"btn btn-primary\"><i class=\"bi bi-floppy me-1\"></i>Update Tags</button></div></form></div></div></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function() {\n\t\t\tinitS3TablesBuckets();\n\t\t});\n\t</script>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 73, "<div class=\"modal fade\" id=\"s3tablesBucketPolicyModal\" tabindex=\"-1\" aria-labelledby=\"s3tablesBucketPolicyModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog modal-xl\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"s3tablesBucketPolicyModalLabel\"><i class=\"bi bi-shield me-2\"></i>Table Bucket Policy</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"s3tablesBucketPolicyForm\"><div class=\"modal-body\"><input type=\"hidden\" id=\"s3tablesBucketPolicyArn\" name=\"bucket_arn\"><ul class=\"nav nav-tabs\" role=\"tablist\"><li class=\"nav-item\" role=\"presentation\"><button class=\"nav-link active\" id=\"s3tablesBucketPolicyEditorTabBtn\" type=\"button\" data-bs-toggle=\"tab\" data-bs-target=\"#s3tablesBucketPolicyEditorTab\" role=\"tab\" aria-controls=\"s3tablesBucketPolicyEditorTab\" aria-selected=\"true\">Editor</button></li><li class=\"nav-item\" role=\"presentation\"><button class=\"nav-link\" id=\"s3tablesBucketPolicyJsonTabBtn\" type=\"button\" data-bs-toggle=\"tab\" data-bs-target=\"#s3tablesBucketPolicyJsonTab\" role=\"tab\" aria-controls=\"s3tablesBucketPolicyJsonTab\" aria-selected=\"false\">JSON</button></li></ul><div class=\"tab-content border border-top-0 rounded-bottom p-3 mb-3\"><div class=\"tab-pane fade show active\" id=\"s3tablesBucketPolicyEditorTab\" role=\"tabpanel\" aria-labelledby=\"s3tablesBucketPolicyEditorTabBtn\"><div id=\"s3tablesBucketPolicyEditorBody\"></div><button type=\"button\" class=\"btn btn-sm btn-outline-secondary\" id=\"s3tablesBucketPolicyAddStatementBtn\"><i class=\"bi bi-plus me-1\"></i>Add statement</button></div><div class=\"tab-pane fade\" id=\"s3tablesBucketPolicyJsonTab\" role=\"tabpanel\" aria-labelledby=\"s3tablesBucketPolicyJsonTabBtn\"><textarea class=\"form-control\" id=\"s3tablesBucketPolicyText\" name=\"policy\" rows=\"12\" placeholder=\"{ }\"></textarea></div></div><div class=\"form-text\">Provide a policy JSON; use Delete Policy to remove the policy.</div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Close</button> <button type=\"button\" class=\"btn btn-outline-danger\" id=\"s3tablesBucketPolicyDeleteBtn\" onclick=\"deleteS3TablesBucketPolicy()\"><i class=\"bi bi-trash me-1\"></i>Delete Policy</button><!-- type=\"button\": a submit button would make Enter in any\n\t\t\t\t\t\t single-line editor input implicitly submit a half-built\n\t\t\t\t\t\t policy, which the backend stores verbatim. --><button type=\"button\" class=\"btn btn-primary\" id=\"s3tablesBucketPolicySaveBtn\"><i class=\"bi bi-floppy me-1\"></i>Save Policy</button></div></form></div></div></div><div class=\"modal fade\" id=\"s3tablesTagsModal\" tabindex=\"-1\" aria-labelledby=\"s3tablesTagsModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog modal-lg\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"s3tablesTagsModalLabel\"><i class=\"bi bi-tags me-2\"></i>Resource Tags</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"s3tablesTagsForm\"><div class=\"modal-body\"><input type=\"hidden\" id=\"s3tablesTagsResourceArn\" name=\"resource_arn\"><div class=\"mb-3\"><label class=\"form-label\">Existing Tags</label><pre class=\"bg-light p-3 border rounded\" id=\"s3tablesTagsList\">Loading...</pre></div><div class=\"mb-3\"><label for=\"s3tablesTagsInput\" class=\"form-label\">Add or Update Tags</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesTagsInput\" placeholder=\"key1=value1,key2=value2\"></div><div class=\"mb-3\"><label for=\"s3tablesTagsDeleteInput\" class=\"form-label\">Remove Tag Keys</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesTagsDeleteInput\" placeholder=\"key1,key2\"></div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Close</button> <button type=\"button\" class=\"btn btn-outline-danger\" onclick=\"deleteS3TablesTags()\"><i class=\"bi bi-trash me-1\"></i>Remove Tags</button> <button type=\"submit\" class=\"btn btn-primary\"><i class=\"bi bi-floppy me-1\"></i>Update Tags</button></div></form></div></div></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function() {\n\t\t\tinitS3TablesBuckets();\n\t\t});\n\t</script>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -0,0 +1,210 @@
package app
import (
"bytes"
"encoding/json"
"fmt"
"strings"
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
)
templ S3VectorsBucketDetails(data dash.S3VectorsBucketData) {
<div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
<div>
<h1 class="h2 mb-0">
<i class="bi bi-bounding-box-circles me-2"></i>{ data.Bucket.Name }
</h1>
<nav aria-label="breadcrumb">
<ol class="breadcrumb mb-0 mt-1">
<li class="breadcrumb-item"><a href={ dash.PUrl(ctx, "/object-store/vector-buckets") }>Vector Buckets</a></li>
<li class="breadcrumb-item active" aria-current="page">{ data.Bucket.Name }</li>
</ol>
</nav>
</div>
</div>
<div class="card shadow mb-4">
<div class="card-body">
<dl class="row mb-0">
<dt class="col-sm-2">ARN</dt>
<dd class="col-sm-10"><code>{ data.Bucket.ARN }</code></dd>
<dt class="col-sm-2">Created</dt>
<dd class="col-sm-10">{ data.Bucket.CreatedAt.Format("2006-01-02 15:04:05") }</dd>
<dt class="col-sm-2">Encryption</dt>
<dd class="col-sm-10">
if data.Bucket.Encryption != "" {
<span class="badge bg-success">{ data.Bucket.Encryption }</span>
} else {
<span class="text-muted">None</span>
}
</dd>
</dl>
</div>
</div>
<div class="card shadow mb-4">
<div class="card-header py-3">
<h6 class="m-0 font-weight-bold text-primary">
<i class="bi bi-list-ul me-2"></i>{ fmt.Sprintf("%d indexes", data.TotalIndexes) }
if data.Truncated {
<span class="text-muted small ms-2">{ fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalIndexes) }</span>
}
</h6>
</div>
<div class="card-body">
<div class="table-responsive">
<table class="table table-hover" width="100%" cellspacing="0" id="s3vectorsIndexesTable">
<thead>
<tr>
<th>Name</th>
<th>Data Type</th>
<th>Dimension</th>
<th>Distance Metric</th>
<th>Non-filterable Metadata Keys</th>
<th>Created</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
for _, index := range data.Indexes {
<tr>
<td>
{ index.IndexName }
<div class="text-muted small">{ index.IndexArn }</div>
</td>
<td>{ index.DataType }</td>
<td>{ fmt.Sprintf("%d", index.Dimension) }</td>
<td>{ index.DistanceMetric }</td>
<td>
if index.MetadataConfiguration != nil && len(index.MetadataConfiguration.NonFilterableMetadataKeys) > 0 {
<code>{ strings.Join(index.MetadataConfiguration.NonFilterableMetadataKeys, ", ") }</code>
} else {
<span class="text-muted">None</span>
}
</td>
<td>{ index.CreationTime.Format("2006-01-02 15:04") }</td>
<td>
<button type="button" class="btn btn-outline-danger btn-sm s3vectors-delete-index-btn" data-index-name={ index.IndexName } title="Delete">
<i class="bi bi-trash"></i>
</button>
</td>
</tr>
}
if len(data.Indexes) == 0 {
<tr>
<td colspan="7" class="text-center text-muted py-4">
No indexes. Create one with an S3 Vectors client, e.g. <code>aws s3vectors create-index</code>.
</td>
</tr>
}
</tbody>
</table>
</div>
</div>
</div>
<div class="card shadow mb-4">
<div class="card-header py-3">
<h6 class="m-0 font-weight-bold text-primary">
<i class="bi bi-shield me-2"></i>Bucket Policy
</h6>
</div>
<div class="card-body">
if data.Policy != "" {
<pre class="bg-light p-3 border rounded mb-0" id="s3vectorsBucketPolicy">{ indentJSON(data.Policy) }</pre>
} else {
<span class="text-muted">No policy. Set one with <code>PutVectorBucketPolicy</code>.</span>
}
</div>
</div>
<div class="modal fade" id="deleteS3VectorsIndexModal" tabindex="-1" aria-labelledby="deleteS3VectorsIndexModalLabel" aria-hidden="true">
<div class="modal-dialog">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="deleteS3VectorsIndexModalLabel">
<i class="bi bi-exclamation-triangle me-2 text-warning"></i>Delete Index
</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
<p>Delete the index <strong id="deleteS3VectorsIndexName"></strong> and all its vectors?</p>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
<button type="button" class="btn btn-danger" id="deleteS3VectorsIndexBtn">
<i class="bi bi-trash me-1"></i>Delete
</button>
</div>
</div>
</div>
</div>
<div id="s3vectorsBucketDetails" data-bucket-name={ data.Bucket.Name }></div>
<script>
document.addEventListener('DOMContentLoaded', function () {
const bucket = document.getElementById('s3vectorsBucketDetails').dataset.bucketName;
const modalEl = document.getElementById('deleteS3VectorsIndexModal');
const modal = new bootstrap.Modal(modalEl);
document.querySelectorAll('.s3vectors-delete-index-btn').forEach(button => {
button.addEventListener('click', function () {
modalEl.dataset.indexName = this.dataset.indexName || '';
document.getElementById('deleteS3VectorsIndexName').textContent = this.dataset.indexName || '';
modal.show();
});
});
document.getElementById('deleteS3VectorsIndexBtn').addEventListener('click', async function () {
const index = modalEl.dataset.indexName;
if (!bucket || !index) return;
try {
const response = await fetch(s3tBasePath('/api/s3vectors/indexes?bucket=' + encodeURIComponent(bucket) + '&index=' + encodeURIComponent(index)), { method: 'DELETE', headers: s3tWriteHeaders() });
const data = await response.json();
if (!response.ok) {
alert(data.error || 'Failed to delete index');
return;
}
location.reload();
} catch (error) {
alert('Failed to delete index: ' + error.message);
}
});
});
</script>
}
const maxIndentedJSONDepth = 32
func indentJSON(s string) string {
if jsonDepth(s) > maxIndentedJSONDepth {
return s
}
var out bytes.Buffer
if json.Indent(&out, []byte(s), "", " ") != nil {
return s
}
return out.String()
}
func jsonDepth(s string) int {
depth, maxDepth := 0, 0
inString, escaped := false, false
for i := 0; i < len(s); i++ {
c := s[i]
switch {
case escaped:
escaped = false
case inString:
if c == '\\' {
escaped = true
} else if c == '"' {
inString = false
}
case c == '"':
inString = true
case c == '{' || c == '[':
depth++
if depth > maxDepth {
maxDepth = depth
}
case c == '}' || c == ']':
depth--
}
}
return maxDepth
}
@@ -0,0 +1,392 @@
// Code generated by templ - DO NOT EDIT.
// templ: version: v0.3.1100
package app
//lint:file-ignore SA4006 This context is only used if a nested component is present.
import "github.com/a-h/templ"
import templruntime "github.com/a-h/templ/runtime"
import (
"bytes"
"encoding/json"
"fmt"
"strings"
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
)
func S3VectorsBucketDetails(data dash.S3VectorsBucketData) templ.Component {
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
return templ_7745c5c3_CtxErr
}
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
if !templ_7745c5c3_IsBuffer {
defer func() {
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
if templ_7745c5c3_Err == nil {
templ_7745c5c3_Err = templ_7745c5c3_BufErr
}
}()
}
ctx = templ.InitializeContext(ctx)
ctx = templ.ClearChildren(ctx)
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom\"><div><h1 class=\"h2 mb-0\"><i class=\"bi bi-bounding-box-circles me-2\"></i>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var2 string
templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.Name)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 16, Col: 69}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "</h1><nav aria-label=\"breadcrumb\"><ol class=\"breadcrumb mb-0 mt-1\"><li class=\"breadcrumb-item\"><a href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var3 string
templ_7745c5c3_Var3, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, "/object-store/vector-buckets")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 20, Col: 89}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var3)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "\">Vector Buckets</a></li><li class=\"breadcrumb-item active\" aria-current=\"page\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var4 string
templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.Name)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 21, Col: 78}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "</li></ol></nav></div></div><div class=\"card shadow mb-4\"><div class=\"card-body\"><dl class=\"row mb-0\"><dt class=\"col-sm-2\">ARN</dt><dd class=\"col-sm-10\"><code>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var5 string
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.ARN)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 30, Col: 49}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "</code></dd><dt class=\"col-sm-2\">Created</dt><dd class=\"col-sm-10\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var6 string
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.CreatedAt.Format("2006-01-02 15:04:05"))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 32, Col: 79}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "</dd><dt class=\"col-sm-2\">Encryption</dt><dd class=\"col-sm-10\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if data.Bucket.Encryption != "" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "<span class=\"badge bg-success\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var7 string
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(data.Bucket.Encryption)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 36, Col: 61}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "</span>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "<span class=\"text-muted\">None</span>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</dd></dl></div></div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-list-ul me-2\"></i>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var8 string
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d indexes", data.TotalIndexes))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 47, Col: 84}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, " ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if data.Truncated {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "<span class=\"text-muted small ms-2\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var9 string
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalIndexes))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 49, Col: 138}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "</span>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "</h6></div><div class=\"card-body\"><div class=\"table-responsive\"><table class=\"table table-hover\" width=\"100%\" cellspacing=\"0\" id=\"s3vectorsIndexesTable\"><thead><tr><th>Name</th><th>Data Type</th><th>Dimension</th><th>Distance Metric</th><th>Non-filterable Metadata Keys</th><th>Created</th><th>Actions</th></tr></thead> <tbody>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
for _, index := range data.Indexes {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "<tr><td>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var10 string
templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(index.IndexName)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 71, Col: 26}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "<div class=\"text-muted small\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var11 string
templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(index.IndexArn)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 72, Col: 55}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "</div></td><td>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var12 string
templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(index.DataType)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 74, Col: 28}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "</td><td>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var13 string
templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", index.Dimension))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 75, Col: 48}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "</td><td>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var14 string
templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(index.DistanceMetric)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 76, Col: 34}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "</td><td>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if index.MetadataConfiguration != nil && len(index.MetadataConfiguration.NonFilterableMetadataKeys) > 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "<code>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var15 string
templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(strings.Join(index.MetadataConfiguration.NonFilterableMetadataKeys, ", "))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 79, Col: 91}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "</code>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "<span class=\"text-muted\">None</span>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "</td><td>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var16 string
templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(index.CreationTime.Format("2006-01-02 15:04"))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 84, Col: 59}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "</td><td><button type=\"button\" class=\"btn btn-outline-danger btn-sm s3vectors-delete-index-btn\" data-index-name=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var17 string
templ_7745c5c3_Var17, templ_7745c5c3_Err = templruntime.NewAttributeExpression(index.IndexName).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 86, Col: 129}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var17)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "\" title=\"Delete\"><i class=\"bi bi-trash\"></i></button></td></tr>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
if len(data.Indexes) == 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "<tr><td colspan=\"7\" class=\"text-center text-muted py-4\">No indexes. Create one with an S3 Vectors client, e.g. <code>aws s3vectors create-index</code>.</td></tr>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "</tbody></table></div></div></div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-shield me-2\"></i>Bucket Policy</h6></div><div class=\"card-body\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if data.Policy != "" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "<pre class=\"bg-light p-3 border rounded mb-0\" id=\"s3vectorsBucketPolicy\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var18 string
templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(indentJSON(data.Policy))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 112, Col: 102}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "</pre>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "<span class=\"text-muted\">No policy. Set one with <code>PutVectorBucketPolicy</code>.</span>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "</div></div><div class=\"modal fade\" id=\"deleteS3VectorsIndexModal\" tabindex=\"-1\" aria-labelledby=\"deleteS3VectorsIndexModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"deleteS3VectorsIndexModalLabel\"><i class=\"bi bi-exclamation-triangle me-2 text-warning\"></i>Delete Index</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p>Delete the index <strong id=\"deleteS3VectorsIndexName\"></strong> and all its vectors?</p></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-danger\" id=\"deleteS3VectorsIndexBtn\"><i class=\"bi bi-trash me-1\"></i>Delete</button></div></div></div></div><div id=\"s3vectorsBucketDetails\" data-bucket-name=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var19 string
templ_7745c5c3_Var19, templ_7745c5c3_Err = templruntime.NewAttributeExpression(data.Bucket.Name).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_bucket_details.templ`, Line: 139, Col: 69}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var19)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "\"></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function () {\n\t\t\tconst bucket = document.getElementById('s3vectorsBucketDetails').dataset.bucketName;\n\t\t\tconst modalEl = document.getElementById('deleteS3VectorsIndexModal');\n\t\t\tconst modal = new bootstrap.Modal(modalEl);\n\t\t\tdocument.querySelectorAll('.s3vectors-delete-index-btn').forEach(button => {\n\t\t\t\tbutton.addEventListener('click', function () {\n\t\t\t\t\tmodalEl.dataset.indexName = this.dataset.indexName || '';\n\t\t\t\t\tdocument.getElementById('deleteS3VectorsIndexName').textContent = this.dataset.indexName || '';\n\t\t\t\t\tmodal.show();\n\t\t\t\t});\n\t\t\t});\n\t\t\tdocument.getElementById('deleteS3VectorsIndexBtn').addEventListener('click', async function () {\n\t\t\t\tconst index = modalEl.dataset.indexName;\n\t\t\t\tif (!bucket || !index) return;\n\t\t\t\ttry {\n\t\t\t\t\tconst response = await fetch(s3tBasePath('/api/s3vectors/indexes?bucket=' + encodeURIComponent(bucket) + '&index=' + encodeURIComponent(index)), { method: 'DELETE', headers: s3tWriteHeaders() });\n\t\t\t\t\tconst data = await response.json();\n\t\t\t\t\tif (!response.ok) {\n\t\t\t\t\t\talert(data.error || 'Failed to delete index');\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\talert('Failed to delete index: ' + error.message);\n\t\t\t\t}\n\t\t\t});\n\t\t});\n\t</script>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
return nil
})
}
const maxIndentedJSONDepth = 32
func indentJSON(s string) string {
if jsonDepth(s) > maxIndentedJSONDepth {
return s
}
var out bytes.Buffer
if json.Indent(&out, []byte(s), "", " ") != nil {
return s
}
return out.String()
}
func jsonDepth(s string) int {
depth, maxDepth := 0, 0
inString, escaped := false, false
for i := 0; i < len(s); i++ {
c := s[i]
switch {
case escaped:
escaped = false
case inString:
if c == '\\' {
escaped = true
} else if c == '"' {
inString = false
}
case c == '"':
inString = true
case c == '{' || c == '[':
depth++
if depth > maxDepth {
maxDepth = depth
}
case c == '}' || c == ']':
depth--
}
}
return maxDepth
}
var _ = templruntime.GeneratedTemplate
+189
View File
@@ -0,0 +1,189 @@
package app
import (
"fmt"
"net/url"
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
)
templ S3VectorsBuckets(data dash.S3VectorsBucketsData) {
<div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
<h1 class="h2">
<i class="bi bi-bounding-box-circles me-2"></i>Vector Buckets
</h1>
<div class="btn-toolbar mb-2 mb-md-0">
<div class="btn-group me-2">
<button type="button" class="btn btn-sm btn-primary" data-bs-toggle="modal" data-bs-target="#createS3VectorsBucketModal">
<i class="bi bi-plus me-1"></i>Create Bucket
</button>
</div>
</div>
</div>
<div class="alert alert-info mb-4">
<i class="bi bi-info-circle me-2"></i>
Vector buckets serve the Amazon S3 Vectors API on the S3 port. Each one is a LANCE table bucket that keeps its indexes in its <code>vector</code> namespace.
Indexes are created and filled with an S3 Vectors client, which needs a running <code>weed-vector</code> engine.
</div>
<div class="card shadow mb-4">
<div class="card-header py-3">
<h6 class="m-0 font-weight-bold text-primary">
<i class="bi bi-bounding-box-circles me-2"></i>{ fmt.Sprintf("%d vector buckets", data.TotalBuckets) }
if data.Truncated {
<span class="text-muted small ms-2">{ fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalBuckets) }</span>
}
</h6>
</div>
<div class="card-body">
<div class="table-responsive">
<table class="table table-hover" width="100%" cellspacing="0" id="s3vectorsBucketsTable">
<thead>
<tr>
<th>Name</th>
<th>ARN</th>
<th>Encryption</th>
<th>Created</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
for _, bucket := range data.Buckets {
<tr>
<td>
<a href={ dash.PUrl(ctx, "/object-store/vector-buckets/"+url.PathEscape(bucket.Name)) }>{ bucket.Name }</a>
</td>
<td class="text-muted small">{ bucket.ARN }</td>
<td>
if bucket.Encryption != "" {
<span class="badge bg-success">{ bucket.Encryption }</span>
} else {
<span class="text-muted">None</span>
}
</td>
<td>{ bucket.CreatedAt.Format("2006-01-02 15:04") }</td>
<td>
<div class="btn-group btn-group-sm" role="group">
<a class="btn btn-outline-primary btn-sm" href={ dash.PUrl(ctx, "/object-store/vector-buckets/"+url.PathEscape(bucket.Name)) } title="Indexes">
<i class="bi bi-folder2-open"></i>
</a>
<button type="button" class="btn btn-outline-danger btn-sm s3vectors-delete-bucket-btn" data-bucket-name={ bucket.Name } title="Delete">
<i class="bi bi-trash"></i>
</button>
</div>
</td>
</tr>
}
if len(data.Buckets) == 0 {
<tr>
<td colspan="5" class="text-center text-muted py-4">
<i class="bi bi-bounding-box-circles icon-3x mb-3 text-muted"></i>
<div>
<h5>No vector buckets found</h5>
</div>
</td>
</tr>
}
</tbody>
</table>
</div>
</div>
</div>
<div class="modal fade" id="createS3VectorsBucketModal" tabindex="-1" aria-labelledby="createS3VectorsBucketModalLabel" aria-hidden="true">
<div class="modal-dialog">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="createS3VectorsBucketModalLabel">
<i class="bi bi-plus me-2"></i>Create Vector Bucket
</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<form id="createS3VectorsBucketForm">
<div class="modal-body">
<div class="mb-3">
<label for="s3vectorsBucketName" class="form-label">Bucket Name</label>
<input type="text" class="form-control" id="s3vectorsBucketName" name="name" placeholder="vector-bucket-name" required/>
<div class="form-text">3 to 63 lowercase letters, digits and hyphens, starting and ending with a letter or digit.</div>
</div>
<div class="form-text">
The bucket is created without encryption. To encrypt it, create it with an S3 Vectors client and <code>encryptionConfiguration</code>, which the S3 gateway checks against its SSE-S3 key.
</div>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
<button type="submit" class="btn btn-primary">
<i class="bi bi-plus me-1"></i>Create
</button>
</div>
</form>
</div>
</div>
</div>
<div class="modal fade" id="deleteS3VectorsBucketModal" tabindex="-1" aria-labelledby="deleteS3VectorsBucketModalLabel" aria-hidden="true">
<div class="modal-dialog">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="deleteS3VectorsBucketModalLabel">
<i class="bi bi-exclamation-triangle me-2 text-warning"></i>Delete Vector Bucket
</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
<p>Delete the vector bucket <strong id="deleteS3VectorsBucketName"></strong>? A bucket that still has indexes is not deleted.</p>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
<button type="button" class="btn btn-danger" id="deleteS3VectorsBucketBtn">
<i class="bi bi-trash me-1"></i>Delete
</button>
</div>
</div>
</div>
</div>
<script>
document.addEventListener('DOMContentLoaded', function () {
const deleteModalEl = document.getElementById('deleteS3VectorsBucketModal');
const deleteModal = new bootstrap.Modal(deleteModalEl);
document.querySelectorAll('.s3vectors-delete-bucket-btn').forEach(button => {
button.addEventListener('click', function () {
deleteModalEl.dataset.bucketName = this.dataset.bucketName || '';
document.getElementById('deleteS3VectorsBucketName').textContent = this.dataset.bucketName || '';
deleteModal.show();
});
});
document.getElementById('deleteS3VectorsBucketBtn').addEventListener('click', async function () {
const bucket = deleteModalEl.dataset.bucketName;
if (!bucket) return;
try {
const response = await fetch(s3tBasePath('/api/s3vectors/buckets?bucket=' + encodeURIComponent(bucket)), { method: 'DELETE', headers: s3tWriteHeaders() });
const data = await response.json();
if (!response.ok) {
alert(data.error || 'Failed to delete vector bucket');
return;
}
location.reload();
} catch (error) {
alert('Failed to delete vector bucket: ' + error.message);
}
});
document.getElementById('createS3VectorsBucketForm').addEventListener('submit', async function (e) {
e.preventDefault();
const name = document.getElementById('s3vectorsBucketName').value.trim();
try {
const response = await fetch(s3tBasePath('/api/s3vectors/buckets'), {
method: 'POST',
headers: s3tWriteHeaders({ 'Content-Type': 'application/json' }),
body: JSON.stringify({ name: name })
});
const data = await response.json();
if (!response.ok) {
alert(data.error || 'Failed to create vector bucket');
return;
}
location.reload();
} catch (error) {
alert('Failed to create vector bucket: ' + error.message);
}
});
});
</script>
}
@@ -0,0 +1,202 @@
// Code generated by templ - DO NOT EDIT.
// templ: version: v0.3.1100
package app
//lint:file-ignore SA4006 This context is only used if a nested component is present.
import "github.com/a-h/templ"
import templruntime "github.com/a-h/templ/runtime"
import (
"fmt"
"net/url"
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
)
func S3VectorsBuckets(data dash.S3VectorsBucketsData) templ.Component {
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
return templ_7745c5c3_CtxErr
}
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
if !templ_7745c5c3_IsBuffer {
defer func() {
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
if templ_7745c5c3_Err == nil {
templ_7745c5c3_Err = templ_7745c5c3_BufErr
}
}()
}
ctx = templ.InitializeContext(ctx)
ctx = templ.ClearChildren(ctx)
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom\"><h1 class=\"h2\"><i class=\"bi bi-bounding-box-circles me-2\"></i>Vector Buckets</h1><div class=\"btn-toolbar mb-2 mb-md-0\"><div class=\"btn-group me-2\"><button type=\"button\" class=\"btn btn-sm btn-primary\" data-bs-toggle=\"modal\" data-bs-target=\"#createS3VectorsBucketModal\"><i class=\"bi bi-plus me-1\"></i>Create Bucket</button></div></div></div><div class=\"alert alert-info mb-4\"><i class=\"bi bi-info-circle me-2\"></i> Vector buckets serve the Amazon S3 Vectors API on the S3 port. Each one is a LANCE table bucket that keeps its indexes in its <code>vector</code> namespace. Indexes are created and filled with an S3 Vectors client, which needs a running <code>weed-vector</code> engine.</div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"bi bi-bounding-box-circles me-2\"></i>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var2 string
templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d vector buckets", data.TotalBuckets))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 31, Col: 104}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, " ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if data.Truncated {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "<span class=\"text-muted small ms-2\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var3 string
templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("showing the first %d; list the rest with an S3 Vectors client", data.TotalBuckets))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 33, Col: 138}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "</span>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "</h6></div><div class=\"card-body\"><div class=\"table-responsive\"><table class=\"table table-hover\" width=\"100%\" cellspacing=\"0\" id=\"s3vectorsBucketsTable\"><thead><tr><th>Name</th><th>ARN</th><th>Encryption</th><th>Created</th><th>Actions</th></tr></thead> <tbody>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
for _, bucket := range data.Buckets {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<tr><td><a href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var4 string
templ_7745c5c3_Var4, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, "/object-store/vector-buckets/"+url.PathEscape(bucket.Name))).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 53, Col: 94}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var4)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var5 string
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 53, Col: 110}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "</a></td><td class=\"text-muted small\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var6 string
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 55, Col: 49}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "</td><td>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if bucket.Encryption != "" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "<span class=\"badge bg-success\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var7 string
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Encryption)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 58, Col: 60}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "</span>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "<span class=\"text-muted\">None</span>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "</td><td>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var8 string
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04"))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 63, Col: 57}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "</td><td><div class=\"btn-group btn-group-sm\" role=\"group\"><a class=\"btn btn-outline-primary btn-sm\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var9 string
templ_7745c5c3_Var9, templ_7745c5c3_Err = templruntime.NewAttributeExpression(dash.PUrl(ctx, "/object-store/vector-buckets/"+url.PathEscape(bucket.Name))).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 66, Col: 134}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var9)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "\" title=\"Indexes\"><i class=\"bi bi-folder2-open\"></i></a> <button type=\"button\" class=\"btn btn-outline-danger btn-sm s3vectors-delete-bucket-btn\" data-bucket-name=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var10 string
templ_7745c5c3_Var10, templ_7745c5c3_Err = templruntime.NewAttributeExpression(bucket.Name).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3vectors_buckets.templ`, Line: 69, Col: 128}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var10)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "\" title=\"Delete\"><i class=\"bi bi-trash\"></i></button></div></td></tr>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
if len(data.Buckets) == 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "<tr><td colspan=\"5\" class=\"text-center text-muted py-4\"><i class=\"bi bi-bounding-box-circles icon-3x mb-3 text-muted\"></i><div><h5>No vector buckets found</h5></div></td></tr>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "</tbody></table></div></div></div><div class=\"modal fade\" id=\"createS3VectorsBucketModal\" tabindex=\"-1\" aria-labelledby=\"createS3VectorsBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"createS3VectorsBucketModalLabel\"><i class=\"bi bi-plus me-2\"></i>Create Vector Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"createS3VectorsBucketForm\"><div class=\"modal-body\"><div class=\"mb-3\"><label for=\"s3vectorsBucketName\" class=\"form-label\">Bucket Name</label> <input type=\"text\" class=\"form-control\" id=\"s3vectorsBucketName\" name=\"name\" placeholder=\"vector-bucket-name\" required><div class=\"form-text\">3 to 63 lowercase letters, digits and hyphens, starting and ending with a letter or digit.</div></div><div class=\"form-text\">The bucket is created without encryption. To encrypt it, create it with an S3 Vectors client and <code>encryptionConfiguration</code>, which the S3 gateway checks against its SSE-S3 key.</div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"submit\" class=\"btn btn-primary\"><i class=\"bi bi-plus me-1\"></i>Create</button></div></form></div></div></div><div class=\"modal fade\" id=\"deleteS3VectorsBucketModal\" tabindex=\"-1\" aria-labelledby=\"deleteS3VectorsBucketModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"deleteS3VectorsBucketModalLabel\"><i class=\"bi bi-exclamation-triangle me-2 text-warning\"></i>Delete Vector Bucket</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p>Delete the vector bucket <strong id=\"deleteS3VectorsBucketName\"></strong>? A bucket that still has indexes is not deleted.</p></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-danger\" id=\"deleteS3VectorsBucketBtn\"><i class=\"bi bi-trash me-1\"></i>Delete</button></div></div></div></div><script>\n\t\tdocument.addEventListener('DOMContentLoaded', function () {\n\t\t\tconst deleteModalEl = document.getElementById('deleteS3VectorsBucketModal');\n\t\t\tconst deleteModal = new bootstrap.Modal(deleteModalEl);\n\t\t\tdocument.querySelectorAll('.s3vectors-delete-bucket-btn').forEach(button => {\n\t\t\t\tbutton.addEventListener('click', function () {\n\t\t\t\t\tdeleteModalEl.dataset.bucketName = this.dataset.bucketName || '';\n\t\t\t\t\tdocument.getElementById('deleteS3VectorsBucketName').textContent = this.dataset.bucketName || '';\n\t\t\t\t\tdeleteModal.show();\n\t\t\t\t});\n\t\t\t});\n\t\t\tdocument.getElementById('deleteS3VectorsBucketBtn').addEventListener('click', async function () {\n\t\t\t\tconst bucket = deleteModalEl.dataset.bucketName;\n\t\t\t\tif (!bucket) return;\n\t\t\t\ttry {\n\t\t\t\t\tconst response = await fetch(s3tBasePath('/api/s3vectors/buckets?bucket=' + encodeURIComponent(bucket)), { method: 'DELETE', headers: s3tWriteHeaders() });\n\t\t\t\t\tconst data = await response.json();\n\t\t\t\t\tif (!response.ok) {\n\t\t\t\t\t\talert(data.error || 'Failed to delete vector bucket');\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\talert('Failed to delete vector bucket: ' + error.message);\n\t\t\t\t}\n\t\t\t});\n\t\t\tdocument.getElementById('createS3VectorsBucketForm').addEventListener('submit', async function (e) {\n\t\t\t\te.preventDefault();\n\t\t\t\tconst name = document.getElementById('s3vectorsBucketName').value.trim();\n\t\t\t\ttry {\n\t\t\t\t\tconst response = await fetch(s3tBasePath('/api/s3vectors/buckets'), {\n\t\t\t\t\t\tmethod: 'POST',\n\t\t\t\t\t\theaders: s3tWriteHeaders({ 'Content-Type': 'application/json' }),\n\t\t\t\t\t\tbody: JSON.stringify({ name: name })\n\t\t\t\t\t});\n\t\t\t\t\tconst data = await response.json();\n\t\t\t\t\tif (!response.ok) {\n\t\t\t\t\t\talert(data.error || 'Failed to create vector bucket');\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\talert('Failed to create vector bucket: ' + error.message);\n\t\t\t\t}\n\t\t\t});\n\t\t});\n\t</script>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
return nil
})
}
var _ = templruntime.GeneratedTemplate
@@ -0,0 +1,80 @@
package app
import (
"bytes"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3vectors"
"github.com/stretchr/testify/require"
)
func TestS3VectorsBucketsRendersBuckets(t *testing.T) {
data := dash.S3VectorsBucketsData{TotalBuckets: 2, Buckets: []dash.S3VectorsBucketSummary{
{Name: "media", ARN: "arn:aws:s3vectors:us-east-1:000000000000:bucket/media", CreatedAt: time.Unix(0, 0)},
{Name: "sealed", ARN: "arn:aws:s3vectors:us-east-1:000000000000:bucket/sealed", Encryption: "AES256", CreatedAt: time.Unix(0, 0)},
}}
var out bytes.Buffer
require.NoError(t, S3VectorsBuckets(data).Render(dash.WithURLPrefix(t.Context(), "/admin"), &out))
page := out.String()
require.Contains(t, page, `href="/admin/object-store/vector-buckets/media"`)
require.Contains(t, page, `data-bucket-name="sealed"`)
require.Contains(t, page, `<span class="badge bg-success">AES256</span>`)
require.Contains(t, page, "2 vector buckets")
require.NotContains(t, page, "No vector buckets found")
require.NotContains(t, page, "showing the first")
data.Truncated = true
out.Reset()
require.NoError(t, S3VectorsBuckets(data).Render(t.Context(), &out))
require.Contains(t, out.String(), "showing the first 2; list the rest with an S3 Vectors client")
}
func TestS3VectorsBucketDetailsRendersIndexesAndPolicy(t *testing.T) {
data := dash.S3VectorsBucketData{
Bucket: dash.S3VectorsBucketSummary{Name: "media", ARN: "arn:aws:s3vectors:us-east-1:000000000000:bucket/media"},
Policy: `{"Version":"2012-10-17","Statement":[]}`,
TotalIndexes: 1,
Indexes: []*s3vectors.Index{{
IndexName: "movies", IndexArn: "arn:aws:s3vectors:us-east-1:000000000000:bucket/media/index/movies",
DataType: "float32", Dimension: 384, DistanceMetric: "cosine",
MetadataConfiguration: &s3vectors.MetadataConfiguration{NonFilterableMetadataKeys: []string{"summary", "body"}},
}},
}
var out bytes.Buffer
require.NoError(t, S3VectorsBucketDetails(data).Render(t.Context(), &out))
page := out.String()
for _, want := range []string{"movies", ">384<", ">cosine<", ">float32<", "summary, body", `data-index-name="movies"`, "{\n &#34;Version&#34;: &#34;2012-10-17&#34;,\n &#34;Statement&#34;: []\n}"} {
require.Contains(t, page, want)
}
require.False(t, strings.Contains(page, "No indexes."), "an index is listed")
data.Policy = "not json"
out.Reset()
require.NoError(t, S3VectorsBucketDetails(data).Render(t.Context(), &out))
require.Contains(t, out.String(), ">not json</pre>")
data.Policy = `{"Version":"2012-10-17","Extra":` + strings.Repeat("[", 5000) + strings.Repeat("]", 5000) + "}"
out.Reset()
require.NoError(t, S3VectorsBucketDetails(data).Render(t.Context(), &out))
require.Contains(t, out.String(), `&#34;Extra&#34;:[[[[`, "a deeply nested policy is shown as stored")
}
func TestJSONDepthSkipsStrings(t *testing.T) {
require.Equal(t, 3, jsonDepth(`{"a":["[{\"[",{}]}`))
require.Equal(t, 0, jsonDepth(`"[[["`))
}
func TestS3TablesBucketsLinksVectorBuckets(t *testing.T) {
data := dash.S3TablesBucketsData{Buckets: []dash.S3TablesBucketSummary{
{Name: "vectors", ARN: "arn:aws:s3tables:us-east-1:000000000000:bucket/vectors", Format: "LANCE"},
{Name: "warehouse", ARN: "arn:aws:s3tables:us-east-1:000000000000:bucket/warehouse", Format: "ICEBERG"},
}}
var out bytes.Buffer
require.NoError(t, S3TablesBuckets(data).Render(t.Context(), &out))
page := out.String()
require.Contains(t, page, `href="/object-store/vector-buckets/vectors"`)
require.NotContains(t, page, `href="/object-store/vector-buckets/warehouse"`)
}
+5
View File
@@ -189,6 +189,11 @@ templ Layout(view ViewContext, content templ.Component) {
<i class="bi bi-table me-2"></i>Table Buckets
</a>
</li>
<li class="nav-item">
<a class="nav-link" href={ view.P("/object-store/vector-buckets") }>
<i class="bi bi-bounding-box-circles me-2"></i>Vector Buckets
</a>
</li>
<li class="nav-item">
<a class="nav-link" href={ view.P("/object-store/users") }>
<i class="bi bi-people me-2"></i>Users
+249 -236
View File
@@ -534,365 +534,378 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var32 string
templ_7745c5c3_Var32, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/users")).ResolveURL(ctx)
templ_7745c5c3_Var32, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/vector-buckets")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 193, Col: 84}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 193, Col: 93}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var32)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "\"><i class=\"bi bi-people me-2\"></i>Users</a></li><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "\"><i class=\"bi bi-bounding-box-circles me-2\"></i>Vector Buckets</a></li><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var33 string
templ_7745c5c3_Var33, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/groups")).ResolveURL(ctx)
templ_7745c5c3_Var33, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/users")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 198, Col: 85}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 198, Col: 84}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var33)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "\"><i class=\"bi bi-people-fill me-2\"></i>Groups</a></li><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "\"><i class=\"bi bi-people me-2\"></i>Users</a></li><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var34 string
templ_7745c5c3_Var34, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/service-accounts")).ResolveURL(ctx)
templ_7745c5c3_Var34, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/groups")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 203, Col: 95}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 203, Col: 85}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var34)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "\"><i class=\"bi bi-robot me-2\"></i>Service Accounts</a></li><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "\"><i class=\"bi bi-people-fill me-2\"></i>Groups</a></li><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var35 string
templ_7745c5c3_Var35, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/policies")).ResolveURL(ctx)
templ_7745c5c3_Var35, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/service-accounts")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 208, Col: 87}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 208, Col: 95}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var35)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "\"><i class=\"bi bi-shield me-2\"></i>Policies</a></li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>MANAGEMENT</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "\"><i class=\"bi bi-robot me-2\"></i>Service Accounts</a></li><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var36 string
templ_7745c5c3_Var36, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/files")).ResolveURL(ctx)
templ_7745c5c3_Var36, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/object-store/policies")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 219, Col: 71}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 213, Col: 87}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var36)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "\"><i class=\"bi bi-folder me-2\"></i>File Browser</a></li><li class=\"nav-item\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "\"><i class=\"bi bi-shield me-2\"></i>Policies</a></li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>MANAGEMENT</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\"><a class=\"nav-link\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var37 string
templ_7745c5c3_Var37, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/files")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 224, Col: 71}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var37)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "\"><i class=\"bi bi-folder me-2\"></i>File Browser</a></li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if isMQPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "<a class=\"nav-link\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#messageQueueSubmenu\" aria-expanded=\"true\" aria-controls=\"messageQueueSubmenu\"><i class=\"bi bi-chat-dots me-2\"></i>Message Queue <i class=\"bi bi-chevron-down ms-auto\"></i></a> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "<a class=\"nav-link\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#messageQueueSubmenu\" aria-expanded=\"true\" aria-controls=\"messageQueueSubmenu\"><i class=\"bi bi-chat-dots me-2\"></i>Message Queue <i class=\"bi bi-chevron-down ms-auto\"></i></a> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "<a class=\"nav-link collapsed\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#messageQueueSubmenu\" aria-expanded=\"false\" aria-controls=\"messageQueueSubmenu\"><i class=\"bi bi-chat-dots me-2\"></i>Message Queue <i class=\"bi bi-chevron-down ms-auto\"></i></a> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "<a class=\"nav-link collapsed\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#messageQueueSubmenu\" aria-expanded=\"false\" aria-controls=\"messageQueueSubmenu\"><i class=\"bi bi-chat-dots me-2\"></i>Message Queue <i class=\"bi bi-chevron-down ms-auto\"></i></a> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
if isMQPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "<div class=\"collapse show\" id=\"messageQueueSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "<div class=\"collapse show\" id=\"messageQueueSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if currentPath == "/mq/brokers" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "<a class=\"nav-link py-2 active\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var37 string
templ_7745c5c3_Var37, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/brokers")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 240, Col: 108}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var37)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "\"><i class=\"bi bi-server me-2\"></i>Brokers</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "<a class=\"nav-link py-2\" href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "<a class=\"nav-link py-2 active\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var38 string
templ_7745c5c3_Var38, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/brokers")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 244, Col: 101}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 245, Col: 108}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var38)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "\"><i class=\"bi bi-server me-2\"></i>Brokers</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "\"><i class=\"bi bi-server me-2\"></i>Brokers</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "</li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if currentPath == "/mq/topics" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "<a class=\"nav-link py-2 active\" href=\"")
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "<a class=\"nav-link py-2\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var39 string
templ_7745c5c3_Var39, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/topics")).ResolveURL(ctx)
templ_7745c5c3_Var39, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/brokers")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 251, Col: 107}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 249, Col: 101}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var39)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "\"><i class=\"bi bi-card-list me-2\"></i>Topics</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "\"><i class=\"bi bi-server me-2\"></i>Brokers</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "<a class=\"nav-link py-2\" href=\"")
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "</li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if currentPath == "/mq/topics" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "<a class=\"nav-link py-2 active\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var40 string
templ_7745c5c3_Var40, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/topics")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 255, Col: 100}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 256, Col: 107}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var40)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "\"><i class=\"bi bi-card-list me-2\"></i>Topics</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "\"><i class=\"bi bi-card-list me-2\"></i>Topics</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "<a class=\"nav-link py-2\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var41 string
templ_7745c5c3_Var41, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/topics")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 260, Col: 100}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var41)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "\"><i class=\"bi bi-card-list me-2\"></i>Topics</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "</li></ul></div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "</li></ul></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "<div class=\"collapse\" id=\"messageQueueSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var41 string
templ_7745c5c3_Var41, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/brokers")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 267, Col: 97}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var41)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "\"><i class=\"bi bi-server me-2\"></i>Brokers</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "<div class=\"collapse\" id=\"messageQueueSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var42 string
templ_7745c5c3_Var42, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/topics")).ResolveURL(ctx)
templ_7745c5c3_Var42, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/brokers")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 272, Col: 96}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 272, Col: 97}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var42)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "\"><i class=\"bi bi-card-list me-2\"></i>Topics</a></li></ul></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "</li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>WORKERS</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if isDefaultWorkerPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "<a class=\"nav-link active\" href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "\"><i class=\"bi bi-server me-2\"></i>Brokers</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var43 string
templ_7745c5c3_Var43, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/default")).ResolveURL(ctx)
templ_7745c5c3_Var43, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/mq/topics")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 289, Col: 97}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 277, Col: 96}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var43)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "\"><i class=\"bi bi-server me-2\"></i>Default</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "\"><i class=\"bi bi-card-list me-2\"></i>Topics</a></li></ul></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "<a class=\"nav-link\" href=\"")
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "</li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>WORKERS</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if isDefaultWorkerPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "<a class=\"nav-link active\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var44 string
templ_7745c5c3_Var44, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/default")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 293, Col: 90}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 294, Col: 97}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var44)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "\"><i class=\"bi bi-server me-2\"></i>Default</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "\"><i class=\"bi bi-server me-2\"></i>Default</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "</li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if isLifecycleWorkerPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "<a class=\"nav-link active\" href=\"")
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "<a class=\"nav-link\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var45 string
templ_7745c5c3_Var45, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/lifecycle")).ResolveURL(ctx)
templ_7745c5c3_Var45, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/default")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 300, Col: 99}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 298, Col: 90}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var45)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, "\"><i class=\"bi bi-hourglass-split me-2\"></i>Lifecycle</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "\"><i class=\"bi bi-server me-2\"></i>Default</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, "<a class=\"nav-link\" href=\"")
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "</li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if isLifecycleWorkerPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, "<a class=\"nav-link active\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var46 string
templ_7745c5c3_Var46, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/lifecycle")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 304, Col: 92}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 305, Col: 99}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var46)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, "\"><i class=\"bi bi-hourglass-split me-2\"></i>Lifecycle</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, "\"><i class=\"bi bi-hourglass-split me-2\"></i>Lifecycle</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 68, "</li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if isIcebergWorkerPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 69, "<a class=\"nav-link active\" href=\"")
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, "<a class=\"nav-link\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var47 string
templ_7745c5c3_Var47, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/iceberg")).ResolveURL(ctx)
templ_7745c5c3_Var47, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/lifecycle")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 311, Col: 97}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 309, Col: 92}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var47)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 70, "\"><i class=\"bi bi-snow me-2\"></i>Iceberg</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 68, "\"><i class=\"bi bi-hourglass-split me-2\"></i>Lifecycle</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 71, "<a class=\"nav-link\" href=\"")
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 69, "</li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if isIcebergWorkerPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 70, "<a class=\"nav-link active\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var48 string
templ_7745c5c3_Var48, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/iceberg")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 315, Col: 90}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 316, Col: 97}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var48)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 72, "\"><i class=\"bi bi-snow me-2\"></i>Iceberg</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 71, "\"><i class=\"bi bi-snow me-2\"></i>Iceberg</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 73, "</li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if isLanceWorkerPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 74, "<a class=\"nav-link active\" href=\"")
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 72, "<a class=\"nav-link\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var49 string
templ_7745c5c3_Var49, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/lance")).ResolveURL(ctx)
templ_7745c5c3_Var49, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/iceberg")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 322, Col: 95}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 320, Col: 90}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var49)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 75, "\"><i class=\"bi bi-bounding-box me-2\"></i>Lance</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 73, "\"><i class=\"bi bi-snow me-2\"></i>Iceberg</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 76, "<a class=\"nav-link\" href=\"")
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 74, "</li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if isLanceWorkerPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 75, "<a class=\"nav-link active\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var50 string
templ_7745c5c3_Var50, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/lance")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 326, Col: 88}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 327, Col: 95}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var50)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 77, "\"><i class=\"bi bi-bounding-box me-2\"></i>Lance</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 76, "\"><i class=\"bi bi-bounding-box me-2\"></i>Lance</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 77, "<a class=\"nav-link\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var51 string
templ_7745c5c3_Var51, templ_7745c5c3_Err = templruntime.NewAttributeExpression(view.P("/plugin/lanes/lance")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 331, Col: 88}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var51)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 78, "\"><i class=\"bi bi-bounding-box me-2\"></i>Lance</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 78, "</li></ul></div></div><!-- Sidebar backdrop for mobile --><div class=\"sidebar-backdrop\" id=\"sidebarBackdrop\"></div><!-- Main content --><main class=\"col-md-9 ms-sm-auto col-lg-10 px-3 px-md-4\"><div class=\"pt-3\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 79, "</li></ul></div></div><!-- Sidebar backdrop for mobile --><div class=\"sidebar-backdrop\" id=\"sidebarBackdrop\"></div><!-- Main content --><main class=\"col-md-9 ms-sm-auto col-lg-10 px-3 px-md-4\"><div class=\"pt-3\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -900,108 +913,95 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 79, "</div></main></div></div><!-- Footer --><footer class=\"footer mt-auto py-3 bg-light\"><div class=\"container-fluid text-center\"><small class=\"text-muted\">&copy; ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var51 string
templ_7745c5c3_Var51, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year()))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 351, Col: 60}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var51))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 80, " SeaweedFS Admin v")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 80, "</div></main></div></div><!-- Footer --><footer class=\"footer mt-auto py-3 bg-light\"><div class=\"container-fluid text-center\"><small class=\"text-muted\">&copy; ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var52 string
templ_7745c5c3_Var52, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER)
templ_7745c5c3_Var52, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year()))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 351, Col: 102}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 356, Col: 60}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var52))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 81, " ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 81, " SeaweedFS Admin v")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var53 string
templ_7745c5c3_Var53, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 356, Col: 102}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var53))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 82, " ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if version.COMMIT != "" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 82, "<span class=\"mx-1\">(")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 83, "<span class=\"mx-1\">(")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var53 string
templ_7745c5c3_Var53, templ_7745c5c3_Err = templ.JoinStringErrs(version.COMMIT)
var templ_7745c5c3_Var54 string
templ_7745c5c3_Var54, templ_7745c5c3_Err = templ.JoinStringErrs(version.COMMIT)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 353, Col: 55}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 358, Col: 55}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var53))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var54))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 83, ")</span> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 84, ")</span> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
if !strings.Contains(version.VERSION, "enterprise") {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 84, "<span class=\"mx-2\">•</span> <a href=\"https://seaweedfs.com\" target=\"_blank\" class=\"text-decoration-none\"><i class=\"bi bi-star me-1\"></i>Enterprise Version Available</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 85, "<span class=\"mx-2\">•</span> <a href=\"https://seaweedfs.com\" target=\"_blank\" class=\"text-decoration-none\"><i class=\"bi bi-star me-1\"></i>Enterprise Version Available</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 85, "</small></div></footer><!-- Bootstrap JS --><script src=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var54 string
templ_7745c5c3_Var54, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/bootstrap.bundle.min.js"))).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 366, Col: 70}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var54)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 86, "\"></script><!-- Modal Alerts JS (replaces native alert/confirm) --><script src=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 86, "</small></div></footer><!-- Bootstrap JS --><script src=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var55 string
templ_7745c5c3_Var55, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/modal-alerts.js"))).ResolveURL(ctx)
templ_7745c5c3_Var55, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/bootstrap.bundle.min.js"))).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 368, Col: 62}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 371, Col: 70}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var55)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 87, "\"></script><!-- Custom JS --><script src=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 87, "\"></script><!-- Modal Alerts JS (replaces native alert/confirm) --><script src=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var56 string
templ_7745c5c3_Var56, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/admin.js"))).ResolveURL(ctx)
templ_7745c5c3_Var56, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/modal-alerts.js"))).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 370, Col: 55}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 373, Col: 62}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var56)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 88, "\"></script><script src=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 88, "\"></script><!-- Custom JS --><script src=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var57 string
templ_7745c5c3_Var57, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/iam-utils.js"))).ResolveURL(ctx)
templ_7745c5c3_Var57, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/admin.js"))).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 371, Col: 59}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 375, Col: 55}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var57)
if templ_7745c5c3_Err != nil {
@@ -1012,9 +1012,9 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var58 string
templ_7745c5c3_Var58, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/s3tables.js"))).ResolveURL(ctx)
templ_7745c5c3_Var58, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/iam-utils.js"))).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 372, Col: 58}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 376, Col: 59}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var58)
if templ_7745c5c3_Err != nil {
@@ -1025,15 +1025,28 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var59 string
templ_7745c5c3_Var59, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/policy_editor.js"))).ResolveURL(ctx)
templ_7745c5c3_Var59, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/s3tables.js"))).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 373, Col: 63}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 377, Col: 58}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var59)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 91, "\"></script></body></html>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 91, "\"></script><script src=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var60 string
templ_7745c5c3_Var60, templ_7745c5c3_Err = templruntime.NewAttributeExpression(string(view.P("/static/js/policy_editor.js"))).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 378, Col: 63}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var60)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 92, "\"></script></body></html>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -1059,134 +1072,134 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon
ctx = templ.InitializeContext(ctx)
ctx = templ.ClearChildren(ctx)
prefix := dash.URLPrefixFromContext(ctx)
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 92, "<!doctype html><html lang=\"en\"><head><meta charset=\"UTF-8\"><title>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var61 string
templ_7745c5c3_Var61, templ_7745c5c3_Err = templ.JoinStringErrs(title)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 386, Col: 17}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var61))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 93, " - Login</title><link rel=\"icon\" href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 93, "<!doctype html><html lang=\"en\"><head><meta charset=\"UTF-8\"><title>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var62 string
templ_7745c5c3_Var62, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/favicon.ico").ResolveURL(ctx)
templ_7745c5c3_Var62, templ_7745c5c3_Err = templ.JoinStringErrs(title)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 387, Col: 58}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 391, Col: 17}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var62)
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var62))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, "\" type=\"image/x-icon\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><link href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, " - Login</title><link rel=\"icon\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var63 string
templ_7745c5c3_Var63, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/css/bootstrap.min.css").ResolveURL(ctx)
templ_7745c5c3_Var63, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/favicon.ico").ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 389, Col: 57}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 392, Col: 58}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var63)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 95, "\" rel=\"stylesheet\"><link href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 95, "\" type=\"image/x-icon\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><link href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var64 string
templ_7745c5c3_Var64, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/css/bootstrap-icons.min.css").ResolveURL(ctx)
templ_7745c5c3_Var64, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/css/bootstrap.min.css").ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 390, Col: 67}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 394, Col: 57}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var64)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, "\" rel=\"stylesheet\"></head><body class=\"bg-light\"><div class=\"container\"><div class=\"row justify-content-center min-vh-100 align-items-center\"><div class=\"col-md-6 col-lg-4\"><div class=\"card shadow\"><div class=\"card-body p-5\"><div class=\"text-center mb-4\"><i class=\"bi bi-server icon-3x text-primary mb-3\"></i><h4 class=\"card-title\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, "\" rel=\"stylesheet\"><link href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var65 string
templ_7745c5c3_Var65, templ_7745c5c3_Err = templ.JoinStringErrs(title)
templ_7745c5c3_Var65, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/css/bootstrap-icons.min.css").ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 400, Col: 57}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 395, Col: 67}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var65))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var65)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 97, "</h4><p class=\"text-muted\">Please sign in to continue</p></div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 97, "\" rel=\"stylesheet\"></head><body class=\"bg-light\"><div class=\"container\"><div class=\"row justify-content-center min-vh-100 align-items-center\"><div class=\"col-md-6 col-lg-4\"><div class=\"card shadow\"><div class=\"card-body p-5\"><div class=\"text-center mb-4\"><i class=\"bi bi-server icon-3x text-primary mb-3\"></i><h4 class=\"card-title\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var66 string
templ_7745c5c3_Var66, templ_7745c5c3_Err = templ.JoinStringErrs(title)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 405, Col: 57}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var66))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 98, "</h4><p class=\"text-muted\">Please sign in to continue</p></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if errorMessage != "" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 98, "<div class=\"alert alert-danger\" role=\"alert\"><i class=\"bi bi-exclamation-triangle me-2\"></i> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 99, "<div class=\"alert alert-danger\" role=\"alert\"><i class=\"bi bi-exclamation-triangle me-2\"></i> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var66 string
templ_7745c5c3_Var66, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage)
var templ_7745c5c3_Var67 string
templ_7745c5c3_Var67, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 407, Col: 45}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 412, Col: 45}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var66))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var67))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 99, "</div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 100, "</div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 100, "<form method=\"POST\" action=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var67 string
templ_7745c5c3_Var67, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.SafeURL(prefix + "/login")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 411, Col: 85}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var67)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 101, "\"><input type=\"hidden\" name=\"csrf_token\" value=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 101, "<form method=\"POST\" action=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var68 string
templ_7745c5c3_Var68, templ_7745c5c3_Err = templruntime.NewAttributeExpression(csrfToken).Resolve(ctx)
templ_7745c5c3_Var68, templ_7745c5c3_Err = templruntime.NewAttributeExpression(templ.SafeURL(prefix + "/login")).ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 412, Col: 84}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 416, Col: 85}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var68)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 102, "\"><div class=\"mb-3\"><label for=\"username\" class=\"form-label\">Username</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"bi bi-person\"></i></span> <input type=\"text\" class=\"form-control\" id=\"username\" name=\"username\" required></div></div><div class=\"mb-4\"><label for=\"password\" class=\"form-label\">Password</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"bi bi-lock\"></i></span> <input type=\"password\" class=\"form-control\" id=\"password\" name=\"password\" required></div></div><button type=\"submit\" class=\"btn btn-primary w-100\"><i class=\"bi bi-box-arrow-in-right me-2\"></i>Sign In</button></form></div></div></div></div></div><script src=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 102, "\"><input type=\"hidden\" name=\"csrf_token\" value=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var69 string
templ_7745c5c3_Var69, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/js/bootstrap.bundle.min.js").ResolveURL(ctx)
templ_7745c5c3_Var69, templ_7745c5c3_Err = templruntime.NewAttributeExpression(csrfToken).Resolve(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 443, Col: 63}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 417, Col: 84}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var69)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 103, "\"></script></body></html>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 103, "\"><div class=\"mb-3\"><label for=\"username\" class=\"form-label\">Username</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"bi bi-person\"></i></span> <input type=\"text\" class=\"form-control\" id=\"username\" name=\"username\" required></div></div><div class=\"mb-4\"><label for=\"password\" class=\"form-label\">Password</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"bi bi-lock\"></i></span> <input type=\"password\" class=\"form-control\" id=\"password\" name=\"password\" required></div></div><button type=\"submit\" class=\"btn btn-primary w-100\"><i class=\"bi bi-box-arrow-in-right me-2\"></i>Sign In</button></form></div></div></div></div></div><script src=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var70 string
templ_7745c5c3_Var70, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/js/bootstrap.bundle.min.js").ResolveURL(ctx)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 448, Col: 63}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var70)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 104, "\"></script></body></html>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
+5
View File
@@ -64,6 +64,7 @@ type AdminOptions struct {
debugPort *int
cpuProfile *string
memProfile *string
uploadMaxSizeMB *int
// workerGrpcListener, when set, is a listener already bound to grpcPort by
// the caller. `weed mini` reserves the port this way because the admin
@@ -125,6 +126,7 @@ func init() {
a.debugPort = cmdAdmin.Flag.Int("debug.port", 6060, "http port for debugging")
a.cpuProfile = cmdAdmin.Flag.String("cpuprofile", "", "cpu profile output file")
a.memProfile = cmdAdmin.Flag.String("memprofile", "", "memory profile output file")
a.uploadMaxSizeMB = cmdAdmin.Flag.Int("uploadMaxSizeMB", dash.DefaultUploadMaxSizeMB, "maximum total size in MB of one file browser upload (0 for no limit)")
}
var cmdAdmin = &Command{
@@ -500,6 +502,9 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,
s3PublicEndpoint = options.defaultS3PublicEndpoint
}
adminServer := dash.NewAdminServer(*options.master, *options.filerGroup, nil, dataDir, icebergPort, lancePort, s3PublicEndpoint)
if options.uploadMaxSizeMB != nil {
adminServer.SetUploadMaxSize(int64(*options.uploadMaxSizeMB) << 20)
}
if err := adminServer.ApplyPluginConfigFromToml(util.GetViper()); err != nil {
return fmt.Errorf("apply admin.toml to plugin config: %w", err)
+35
View File
@@ -1,11 +1,14 @@
package command
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/replication/sink"
"github.com/seaweedfs/seaweedfs/weed/replication/sink/localsink"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/seaweedfs/seaweedfs/weed/util"
)
@@ -343,3 +346,35 @@ func TestGenProcessFunctionRenameFromExcludedDirBecomesCreate(t *testing.T) {
t.Fatalf("unexpected delete/update calls: deletes=%v updates=%v", dataSink.deleteKeys, dataSink.updateKeys)
}
}
func TestGenProcessFunctionLocalSinkStaysInsideTarget(t *testing.T) {
root := t.TempDir()
target := filepath.Join(root, "backup")
dataSink := &localsink.LocalSink{Dir: target}
dataSink.SetSourceFiler(&source.FilerSource{})
processFn := genProcessFunction("/", target, nil, nil, nil, nil, dataSink, true, false)
for _, dir := range []string{"/safe", "/../escape"} {
err := processFn(&filer_pb.SubscribeMetadataResponse{
Directory: dir,
EventNotification: &filer_pb.EventNotification{
NewParentPath: dir,
NewEntry: &filer_pb.Entry{
Name: "marker.txt",
Content: []byte("marker"),
Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6},
},
},
})
if err != nil {
t.Fatalf("processFn %s: %v", dir, err)
}
}
if _, err := os.Stat(filepath.Join(target, "safe", "marker.txt")); err != nil {
t.Fatalf("inside file missing: %v", err)
}
if _, err := os.Stat(filepath.Join(root, "escape")); !os.IsNotExist(err) {
t.Fatalf("local sink wrote outside %s: %v", target, err)
}
}
+2
View File
@@ -17,6 +17,7 @@ import (
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -598,6 +599,7 @@ func initMiniAdminFlags() {
miniAdminOptions.allowInsecureBind = cmdMini.Flag.Bool("admin.allowInsecureBind", false, "INSECURE: allow the unauthenticated admin interface to bind to -ip.bind instead of loopback")
miniAdminWorkerBindIP = cmdMini.Flag.String("admin.worker.ip", "127.0.0.1", "worker gRPC bind address; expose only with grpc.admin mTLS configured")
miniAdminOptions.urlPrefix = cmdMini.Flag.String("admin.urlPrefix", "", "URL path prefix when running the admin UI behind a reverse proxy under a subdirectory (e.g. /seaweedfs)")
miniAdminOptions.uploadMaxSizeMB = cmdMini.Flag.Int("admin.uploadMaxSizeMB", dash.DefaultUploadMaxSizeMB, "maximum total size in MB of one file browser upload (0 for no limit)")
}
func init() {
+2
View File
@@ -675,6 +675,7 @@ func (s3opt *S3Options) startLanceServer(s3ApiServer *s3api.S3ApiServer, listenO
lanceServer := lance.NewServer(s3ApiServer, s3ApiServer)
lanceServer.SetCredentialValidator(s3ApiServer)
lanceServer.SetSigningKey(s3ApiServer.FilerSigningKey)
if s3opt.icebergCredentialRole != nil && *s3opt.icebergCredentialRole != "" {
lanceServer.SetCredentialVendor(lanceCredentialVendor{s3ApiServer})
}
@@ -723,6 +724,7 @@ func (s3opt *S3Options) vectorsRouter(s3ApiServer *s3api.S3ApiServer, grpcDialOp
vectorsServer := s3vectors.NewServer(s3ApiServer, s3ApiServer, backend)
vectorsServer.SetRegion(s3tables.DefaultRegion)
vectorsServer.SetAccountID(s3tables.DefaultAccountID)
vectorsServer.SetEncryptionChecker(s3api.GetSSES3KeyManager().IsConfigured)
vectorsServer.RegisterRoutes(vectorsRouter)
return vectorsRouter
}
+7
View File
@@ -501,6 +501,10 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
return filer_pb.ErrEntryNameTooLong
}
if entry.FullPath.HasDotSegment() {
return fmt.Errorf("invalid path %s", entry.FullPath)
}
if entry.IsDirectory() {
entry.Attr.TtlSec = 0
}
@@ -794,6 +798,9 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath,
}
func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry, isFromOtherCluster bool) (err error) {
if entry.FullPath.HasDotSegment() {
return fmt.Errorf("invalid path %s", entry.FullPath)
}
ctx = f.syncWriteContext(ctx, entry.FullPath)
if oldEntry != nil {
entry.Attr.Crtime = oldEntry.Attr.Crtime
+1 -1
View File
@@ -10,7 +10,7 @@ import (
func (f *Filer) CanRename(ctx context.Context, source, target util.FullPath, oldName string) error {
sourcePath := source.Child(oldName)
if strings.HasPrefix(string(target), string(sourcePath)) {
if util.IsEqualOrUnder(string(target), string(sourcePath)) {
return fmt.Errorf("mv: can not move directory to a subdirectory of itself")
}
+39
View File
@@ -1,6 +1,9 @@
package filer
import (
"context"
"os"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/util"
@@ -29,3 +32,39 @@ func TestDetectBucket(t *testing.T) {
})
}
}
func TestCanRenameIntoItself(t *testing.T) {
f, store := newTestFilerWithStubStore(t)
f.DirBucketsPath = "/buckets"
if err := store.InsertEntry(context.Background(), &Entry{FullPath: "/dir/a", Attr: Attr{Mode: os.ModeDir}}); err != nil {
t.Fatal(err)
}
tests := []struct {
name string
oldName string
target string
reject bool
}{
{"into itself", "a", "/dir/a", true},
{"into a child", "a", "/dir/a/b", true},
{"into a deeper descendant", "a", "/dir/a/b/c", true},
{"into a child with a trailing slash", "a", "/dir/a/b/", true},
{"old name with a trailing slash into a child", "a/", "/dir/a/b", true},
{"into a sibling sharing the name prefix", "a", "/dir/a.d", false},
{"into a descendant of that sibling", "a", "/dir/ab/c", false},
{"into the parent", "a", "/dir", false},
{"into an unrelated directory", "a", "/other", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := f.CanRename(context.Background(), "/dir", util.FullPath(tt.target), tt.oldName)
rejected := err != nil && strings.Contains(err.Error(), "subdirectory of itself")
if rejected != tt.reject {
t.Errorf("CanRename(/dir, %q, %q) = %v, want rejected=%v", tt.target, tt.oldName, err, tt.reject)
}
if !tt.reject && err != nil {
t.Errorf("CanRename(/dir, %q, %q) = %v, want nil", tt.target, tt.oldName, err)
}
})
}
}
+10 -3
View File
@@ -8,6 +8,7 @@ import (
"io"
"math"
"os"
"path/filepath"
"strings"
"sync"
@@ -167,9 +168,7 @@ func (store *LevelDB3Store) findDB(fullpath weed_util.FullPath, isForChildren bo
shortPath = weed_util.FullPath(bucketAndObjectKey[t:])
}
// Dot-prefixed entries directly under /buckets (e.g. .system) are internal
// folders, not S3 buckets; keep them in the default DB by full path.
if strings.HasPrefix(bucket, ".") {
if !isBucketFolder(bucket) {
store.dbsLock.RUnlock()
return defaultDB, DEFAULT, fullpath, nil
}
@@ -207,6 +206,14 @@ func (store *LevelDB3Store) emptyBucketDB() (*leveldb.DB, error) {
return store.emptyDB, nil
}
// Dot-prefixed entries directly under /buckets (e.g. .system) are internal
// folders, not S3 buckets; they stay in the default DB by full path. A bucket
// also names a single folder under store.dir, so it cannot hold a separator;
// "\" is an ordinary name character on Unix and only rejected where it separates.
func isBucketFolder(bucket string) bool {
return !strings.HasPrefix(bucket, ".") && filepath.IsLocal(bucket) && !strings.ContainsAny(bucket, "/"+string(os.PathSeparator))
}
func (store *LevelDB3Store) createDB(bucket string) (*leveldb.DB, error) {
store.dbsLock.Lock()
+7 -3
View File
@@ -9,14 +9,18 @@ import (
var _ filer.BucketAware = (*LevelDB3Store)(nil)
func (store *LevelDB3Store) OnBucketCreation(bucket string) {
if !isBucketFolder(bucket) {
return
}
store.createDB(bucket)
}
func (store *LevelDB3Store) OnBucketDeletion(bucket string) {
store.closeDB(bucket)
if bucket != "" { // just to make sure
os.RemoveAll(store.dir + "/" + bucket)
if !isBucketFolder(bucket) || bucket == DEFAULT {
return
}
store.closeDB(bucket)
os.RemoveAll(store.dir + "/" + bucket)
}
func (store *LevelDB3Store) CanDropWholeBucket() bool {
@@ -3,6 +3,8 @@ package leveldb
import (
"context"
"math"
"os"
"path/filepath"
"strings"
"testing"
@@ -207,3 +209,44 @@ func TestListDirectoryPrefixedEntries_StartBelowPrefix(t *testing.T) {
t.Fatalf("want [item-b.versions], got %v", names)
}
}
func TestBucketEventsStayInsideStoreDir(t *testing.T) {
parent := t.TempDir()
dir := filepath.Join(parent, "store")
store := &LevelDB3Store{}
if err := store.initialize(dir); err != nil {
t.Fatal(err)
}
defer store.Shutdown()
sentinel := filepath.Join(parent, "keep")
if err := os.WriteFile(sentinel, []byte("keep"), 0644); err != nil {
t.Fatal(err)
}
// Checked after each create, before the matching delete could remove the
// evidence of a db opened in the wrong place.
assertNothingEscaped := func(bucket string) {
t.Helper()
for _, p := range []string{filepath.Join(parent, "outside"), filepath.Join(parent, "CURRENT"), filepath.Join(dir, "x", "y")} {
if _, err := os.Stat(p); !os.IsNotExist(err) {
t.Fatalf("bucket %q event created %s: %v", bucket, p, err)
}
}
}
store.OnBucketCreation("x")
for _, bucket := range []string{"..", ".", "../outside", "", DEFAULT, "x/../" + DEFAULT, "x/y"} {
store.OnBucketCreation(bucket)
assertNothingEscaped(bucket)
store.OnBucketDeletion(bucket)
}
if _, err := os.Stat(sentinel); err != nil {
t.Fatalf("bucket events removed %s: %v", sentinel, err)
}
if _, err := os.Stat(filepath.Join(dir, DEFAULT)); err != nil {
t.Fatalf("default db folder missing: %v", err)
}
ctx := context.Background()
if err := store.InsertEntry(ctx, &filer.Entry{FullPath: "/a/b"}); err != nil {
t.Fatalf("default db unusable after bucket events: %v", err)
}
}
+63 -15
View File
@@ -8,6 +8,7 @@ import (
"io"
"net/http"
"sort"
"strings"
"sync"
"google.golang.org/grpc"
@@ -94,7 +95,7 @@ func (cm *ChunkManifest) DeleteChunks(masterFn GetMasterFn, usePublicUrl bool, g
return nil
}
func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (written int64, e error) {
func readChunkNeedle(fileUrl string, w io.Writer, offset int64, size int64, jwt string) (written int64, e error) {
req, err := http.NewRequest(http.MethodGet, fileUrl, nil)
if err != nil {
return written, err
@@ -125,7 +126,11 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (wri
return written, util_http.StatusError(fileUrl, resp)
}
return io.Copy(w, resp.Body)
written, e = io.CopyN(w, resp.Body, size)
if e == io.EOF {
e = fmt.Errorf("%s: %d of %d bytes: %w", fileUrl, written, size, io.ErrUnexpectedEOF)
}
return written, e
}
func NewChunkedFileReader(chunkList []*ChunkInfo, master pb.ServerAddress, grpcDialOption grpc.DialOption) *ChunkedFileReader {
@@ -175,26 +180,69 @@ func (cf *ChunkedFileReader) WriteTo(w io.Writer) (n int64, err error) {
return n, ErrInvalidRange
}
for ; chunkIndex < len(cf.chunkList); chunkIndex++ {
ci := cf.chunkList[chunkIndex]
// if we need read date from local volume server first?
fileUrl, jwt, lookupError := LookupFileId(func(_ context.Context) pb.ServerAddress {
return cf.master
}, cf.grpcDialOption, ci.Fid)
if lookupError != nil {
return n, lookupError
}
if wn, e := readChunkNeedle(fileUrl, w, chunkStartOffset, jwt); e != nil {
wn, e := cf.readChunk(cf.chunkList[chunkIndex], w, chunkStartOffset)
n += wn
cf.pos += wn
if e != nil {
return n, e
} else {
n += wn
cf.pos += wn
}
chunkStartOffset = 0
}
return n, nil
}
// readChunk reads the chunk from the first replica that serves it. A failure
// drops the volume's cached locations, and once every cached replica has
// failed the master is asked again. A replica that already sent part of the
// chunk is not continued elsewhere: a volume server streams a needle before
// its checksum is verified, so those bytes may be the corrupt ones.
func (cf *ChunkedFileReader) readChunk(ci *ChunkInfo, w io.Writer, offset int64) (written int64, err error) {
masterFn := func(_ context.Context) pb.ServerAddress {
return cf.master
}
vid, _, _ := strings.Cut(ci.Fid, ",")
dst := &writeErrorRecorder{Writer: w}
tried := make(map[string]bool)
for range 2 {
fileUrls, jwt, lookupError := lookupFileIdUrls(masterFn, cf.grpcDialOption, ci.Fid)
if lookupError != nil {
if err == nil {
err = lookupError
}
return 0, err
}
for _, fileUrl := range fileUrls {
if tried[fileUrl] {
continue
}
tried[fileUrl] = true
written, err = readChunkNeedle(fileUrl, dst, offset, ci.Size-offset, jwt)
if err == nil || dst.err != nil {
return written, err
}
glog.V(0).Infof("read chunk %s from %s: %v", ci.Fid, fileUrl, err)
InvalidateVolumeIdLocationCache(vid)
if written > 0 {
return written, err
}
}
}
return 0, err
}
type writeErrorRecorder struct {
io.Writer
err error
}
func (r *writeErrorRecorder) Write(p []byte) (int, error) {
n, err := r.Writer.Write(p)
if err != nil {
r.err = err
}
return n, err
}
func (cf *ChunkedFileReader) ReadAt(p []byte, off int64) (n int, err error) {
cf.Seek(off, 0)
return cf.Read(p)
+257
View File
@@ -0,0 +1,257 @@
package operation
import (
"bytes"
"context"
"errors"
"io"
"net"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
type fakeVolumeLocationServer struct {
master_pb.UnimplementedSeaweedServer
mu sync.Mutex
calls int
urls []string
}
func (s *fakeVolumeLocationServer) LookupVolume(_ context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.calls++
resp := &master_pb.LookupVolumeResponse{}
for _, vid := range req.VolumeOrFileIds {
var locs []*master_pb.Location
for _, u := range s.urls {
locs = append(locs, &master_pb.Location{Url: u})
}
resp.VolumeIdLocations = append(resp.VolumeIdLocations, &master_pb.LookupVolumeResponse_VolumeIdLocation{
VolumeOrFileId: vid,
Locations: locs,
})
}
return resp, nil
}
func (s *fakeVolumeLocationServer) lookups() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.calls
}
type chunkReplica struct {
*httptest.Server
mu sync.Mutex
ranges []string
}
func (c *chunkReplica) host() string {
return strings.TrimPrefix(c.URL, "http://")
}
func (c *chunkReplica) requests() []string {
c.mu.Lock()
defer c.mu.Unlock()
return append([]string(nil), c.ranges...)
}
// newChunkReplica serves data; with cutAfter >= 0 it announces the full length
// but drops the connection after cutAfter bytes.
func newChunkReplica(t *testing.T, data []byte, cutAfter int) *chunkReplica {
c := &chunkReplica{}
c.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c.mu.Lock()
c.ranges = append(c.ranges, r.Header.Get("Range"))
c.mu.Unlock()
if cutAfter < 0 {
http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(data))
return
}
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
w.WriteHeader(http.StatusOK)
w.Write(data[:cutAfter])
w.(http.Flusher).Flush()
conn, _, err := w.(http.Hijacker).Hijack()
if err == nil {
conn.Close()
}
}))
t.Cleanup(c.Close)
return c
}
func deadAddress(t *testing.T) string {
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
addr := lis.Addr().String()
lis.Close()
return addr
}
func testChunkData(size int) []byte {
data := make([]byte, size)
for i := range data {
data[i] = byte(i * 7)
}
return data
}
func TestChunkedFileReaderTriesTheNextReplica(t *testing.T) {
const vid = "91201"
data := testChunkData(4096)
healthy := newChunkReplica(t, data, -1)
fake := &fakeVolumeLocationServer{urls: []string{healthy.host()}}
master := startFakeMasterServer(t, fake)
dial := grpc.WithTransportCredentials(insecure.NewCredentials())
vc.Set(vid, []Location{{Url: deadAddress(t)}, {Url: healthy.host(), DataInRemote: true}}, time.Minute)
t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) })
chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}}
reader := NewChunkedFileReader(chunks, master, dial)
reader.Seek(1000, io.SeekStart)
got, err := io.ReadAll(reader)
reader.Close()
if err != nil {
t.Fatalf("read: %v", err)
}
if !bytes.Equal(got, data[1000:]) {
t.Fatalf("read %d bytes, want the last %d bytes of the chunk", len(got), len(data)-1000)
}
if r := healthy.requests(); len(r) != 1 || r[0] != "bytes=1000-" {
t.Fatalf("healthy replica requests = %q, want one from byte 1000", r)
}
if _, err := vc.Get(vid); err == nil {
t.Fatalf("cached locations kept after a replica failed")
}
if fake.lookups() != 0 {
t.Fatalf("master lookups = %d, want 0", fake.lookups())
}
}
func TestChunkedFileReaderDoesNotContinueAPartlySentChunk(t *testing.T) {
const vid = "91205"
data := testChunkData(4096)
broken := newChunkReplica(t, data, 1000)
healthy := newChunkReplica(t, data, -1)
fake := &fakeVolumeLocationServer{urls: []string{healthy.host()}}
master := startFakeMasterServer(t, fake)
dial := grpc.WithTransportCredentials(insecure.NewCredentials())
vc.Set(vid, []Location{{Url: broken.host()}, {Url: healthy.host(), DataInRemote: true}}, time.Minute)
t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) })
chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}}
reader := NewChunkedFileReader(chunks, master, dial)
_, err := io.ReadAll(reader)
reader.Close()
if err == nil {
t.Fatalf("read of a chunk cut off mid-body succeeded")
}
if r := healthy.requests(); len(r) != 0 {
t.Fatalf("healthy replica requests = %q, want none", r)
}
reader = NewChunkedFileReader(chunks, master, dial)
got, err := io.ReadAll(reader)
reader.Close()
if err != nil || !bytes.Equal(got, data) {
t.Fatalf("second read: %d bytes, err %v", len(got), err)
}
if fake.lookups() != 1 || len(broken.requests()) != 1 {
t.Fatalf("second read: %d master lookups, %d requests to the broken replica; want 1 and 1", fake.lookups(), len(broken.requests()))
}
}
func TestChunkedFileReaderAsksMasterWhenCachedReplicasFail(t *testing.T) {
const vid = "91202"
first, second := testChunkData(3000), testChunkData(2000)
healthy := newChunkReplica(t, second, -1)
fake := &fakeVolumeLocationServer{urls: []string{healthy.host()}}
master := startFakeMasterServer(t, fake)
dial := grpc.WithTransportCredentials(insecure.NewCredentials())
firstReplica := newChunkReplica(t, first, -1)
const otherVid = "91203"
vc.Set(otherVid, []Location{{Url: firstReplica.host()}}, time.Minute)
vc.Set(vid, []Location{{Url: deadAddress(t)}}, time.Minute)
t.Cleanup(func() {
InvalidateVolumeIdLocationCache(vid)
InvalidateVolumeIdLocationCache(otherVid)
})
chunks := []*ChunkInfo{
{Fid: otherVid + ",01637037d6", Offset: 0, Size: int64(len(first))},
{Fid: vid + ",02637037d6", Offset: int64(len(first)), Size: int64(len(second))},
}
reader := NewChunkedFileReader(chunks, master, dial)
defer reader.Close()
got, err := io.ReadAll(reader)
if err != nil {
t.Fatalf("read: %v", err)
}
if want := append(append([]byte(nil), first...), second...); !bytes.Equal(got, want) {
t.Fatalf("read %d bytes, want %d", len(got), len(want))
}
if fake.lookups() != 1 {
t.Fatalf("master lookups = %d, want 1", fake.lookups())
}
}
func TestChunkedFileReaderTreatsAShortBodyAsFailed(t *testing.T) {
const vid = "91206"
data := testChunkData(2048)
empty := newChunkReplica(t, nil, -1)
healthy := newChunkReplica(t, data, -1)
vc.Set(vid, []Location{{Url: empty.host()}, {Url: healthy.host(), DataInRemote: true}}, time.Minute)
t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) })
chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}}
reader := NewChunkedFileReader(chunks, pb.ServerAddress("127.0.0.1:1"), grpc.WithTransportCredentials(insecure.NewCredentials()))
defer reader.Close()
got, err := io.ReadAll(reader)
if err != nil || !bytes.Equal(got, data) {
t.Fatalf("read: %d bytes, err %v; want %d bytes", len(got), err, len(data))
}
if len(empty.requests()) != 1 || len(healthy.requests()) != 1 {
t.Fatalf("requests: empty %d, healthy %d; want 1 and 1", len(empty.requests()), len(healthy.requests()))
}
}
type failingWriter struct{}
func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("client gone") }
func TestChunkedFileReaderStopsWhenTheClientGoesAway(t *testing.T) {
const vid = "91204"
data := testChunkData(1024)
a, b := newChunkReplica(t, data, -1), newChunkReplica(t, data, -1)
vc.Set(vid, []Location{{Url: a.host()}, {Url: b.host()}}, time.Minute)
t.Cleanup(func() { InvalidateVolumeIdLocationCache(vid) })
chunks := []*ChunkInfo{{Fid: vid + ",01637037d6", Offset: 0, Size: int64(len(data))}}
reader := NewChunkedFileReader(chunks, pb.ServerAddress("127.0.0.1:1"), grpc.WithTransportCredentials(insecure.NewCredentials()))
if _, err := reader.WriteTo(failingWriter{}); err == nil || err.Error() != "client gone" {
t.Fatalf("WriteTo error = %v, want the writer's error", err)
}
if n := len(a.requests()) + len(b.requests()); n != 1 {
t.Fatalf("replica requests = %d, want 1", n)
}
if _, err := vc.Get(vid); err != nil {
t.Fatalf("a client write error dropped the cached locations: %v", err)
}
}
+22 -12
View File
@@ -51,31 +51,41 @@ var (
// authorization the master stamped on the volume; pass it through to the
// volume server on the read request.
func LookupFileId(masterFn GetMasterFn, grpcDialOption grpc.DialOption, fileId string) (fullUrl string, jwt string, err error) {
fullUrls, jwt, err := lookupFileIdUrls(masterFn, grpcDialOption, fileId)
if err != nil {
return "", jwt, err
}
return fullUrls[0], jwt, nil
}
// lookupFileIdUrls returns the read URLs of every replica of fileId, local
// replicas first, each group in random order.
func lookupFileIdUrls(masterFn GetMasterFn, grpcDialOption grpc.DialOption, fileId string) (fullUrls []string, jwt string, err error) {
parts := strings.Split(fileId, ",")
if len(parts) != 2 {
return "", jwt, errors.New("Invalid fileId " + fileId)
return nil, jwt, errors.New("Invalid fileId " + fileId)
}
lookup, lookupError := LookupVolumeId(masterFn, grpcDialOption, parts[0])
if lookupError != nil {
return "", jwt, lookupError
return nil, jwt, lookupError
}
if len(lookup.Locations) == 0 {
return "", jwt, errors.New("File Not Found")
return nil, jwt, errors.New("File Not Found")
}
localUrls := make([]string, 0, len(lookup.Locations))
var localUrls, remoteUrls []string
for _, loc := range lookup.Locations {
if !loc.DataInRemote {
localUrls = append(localUrls, loc.Url)
}
}
if len(localUrls) == 0 {
for _, loc := range lookup.Locations {
localUrls = append(localUrls, loc.Url)
fullUrl := "http://" + loc.Url + "/" + fileId
if loc.DataInRemote {
remoteUrls = append(remoteUrls, fullUrl)
} else {
localUrls = append(localUrls, fullUrl)
}
}
rand.Shuffle(len(localUrls), func(i, j int) { localUrls[i], localUrls[j] = localUrls[j], localUrls[i] })
rand.Shuffle(len(remoteUrls), func(i, j int) { remoteUrls[i], remoteUrls[j] = remoteUrls[j], remoteUrls[i] })
return "http://" + localUrls[rand.IntN(len(localUrls))] + "/" + fileId, lookup.Jwt, nil
return append(localUrls, remoteUrls...), lookup.Jwt, nil
}
func LookupVolumeId(masterFn GetMasterFn, grpcDialOption grpc.DialOption, vid string) (*LookupResult, error) {
+49
View File
@@ -5,6 +5,7 @@ import (
"context"
"fmt"
"io"
"math/rand"
"net/http"
"testing"
@@ -118,6 +119,54 @@ func TestCreateNeedleFromRequest(t *testing.T) {
}
func TestUploadKeepsMarginalGzipUncompressed(t *testing.T) {
data := make([]byte, 4300)
rand.New(rand.NewSource(1)).Read(data[:4000])
gzipped, _ := util.GzipData(data)
if len(gzipped) >= len(data) || len(gzipped)*10 < len(data)*9 {
t.Fatalf("gzip size %d of %d is not a marginal saving", len(gzipped), len(data))
}
mockClient := &MockClient{}
var stored *needle.Needle
mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) {
assert.Equal(t, nil, err, "upload: %v", err)
stored = n
}
uploadOption := &UploadOption{
UploadUrl: "http://localhost:8080/389,0f084d17353afda0",
Filename: "doc.pdf",
MimeType: "application/pdf",
}
newUploader(mockClient).Upload(context.Background(), bytes.NewReader(data), uploadOption)
if stored == nil {
t.Fatal("no needle reached the volume server")
}
assert.False(t, stored.IsCompressed(), "gzip saved less than 10%, data should stay uncompressed")
assert.Equal(t, data, stored.Data)
}
func TestReplicationKeepsTheSourceMime(t *testing.T) {
for _, mimeType := range []string{"", "application/octet-stream", "Text/Plain; charset=UTF-8"} {
mockClient := &MockClient{}
var stored *needle.Needle
mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) {
assert.Equal(t, nil, err, "upload: %v", err)
stored = n
}
newUploader(mockClient).UploadData(context.Background(), []byte(textContent), &UploadOption{
UploadUrl: "http://localhost:8080/389,0f084d17353afda0?type=replicate",
Filename: "report.html",
MimeType: mimeType,
IsReplication: true,
})
if stored == nil {
t.Fatal("no needle reached the volume server")
}
assert.Equal(t, mimeType, string(stored.Mime))
}
}
var textContent = `Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
+4 -3
View File
@@ -390,7 +390,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
option.MimeType = ""
}
}
if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(option.Filename), option.MimeType); iAmSure && shouldBeCompressed {
if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Ext(option.Filename), option.MimeType); iAmSure && shouldBeCompressed {
shouldGzipNow = true
} else if !iAmSure && option.MimeType == "" && len(data) > 16*1024 {
var compressed []byte
@@ -412,7 +412,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
compressed, compressErr := util.GzipData(data)
// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
if compressErr == nil {
if len(compressed) < len(data) {
if len(compressed)*10 < len(data)*9 {
data = compressed
contentIsGzipped = true
}
@@ -474,6 +474,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
Filename: option.Filename,
Cipher: false,
IsInputCompressed: contentIsGzipped,
IsReplication: option.IsReplication,
MimeType: option.MimeType,
PairMap: option.PairMap,
Jwt: option.Jwt,
@@ -510,7 +511,7 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction
// properly handling non-ASCII characters and special characters
h.Set("Content-Disposition", mime.FormatMediaType("form-data", map[string]string{"name": "file", "filename": option.Filename}))
h.Set("Idempotency-Key", option.UploadUrl)
if option.MimeType == "" {
if option.MimeType == "" && !option.IsReplication {
option.MimeType = mime.TypeByExtension(strings.ToLower(filepath.Ext(option.Filename)))
}
if option.MimeType != "" {
+63 -16
View File
@@ -3,6 +3,7 @@ package localsink
import (
"context"
"fmt"
"math/rand/v2"
"os"
"path/filepath"
"strings"
@@ -39,6 +40,34 @@ func (localsink *LocalSink) isMultiPartEntry(key string) bool {
return strings.HasSuffix(key, ".part") && strings.Contains(key, "/"+s3_constants.MultipartUploadsFolder+"/")
}
func (localsink *LocalSink) isInsideDir(key string) (string, bool) {
rel, err := filepath.Rel(filepath.Clean(localsink.Dir), filepath.Clean(key))
if err == nil && filepath.IsLocal(rel) {
return rel, true
}
glog.Warningf("local sink: skip %s outside %s", key, localsink.Dir)
return "", false
}
// resolveInsideDir resolves symlinks in the parent of rel while it stays inside the sink directory.
func (localsink *LocalSink) resolveInsideDir(rel string) string {
base, err := filepath.EvalSymlinks(localsink.Dir)
if err != nil {
return rel
}
dir, rest := filepath.Dir(rel), filepath.Base(rel)
for dir != "." {
if resolved, err := filepath.EvalSymlinks(filepath.Join(base, dir)); err == nil {
if inside, err := filepath.Rel(base, resolved); err == nil && filepath.IsLocal(inside) {
return filepath.Join(inside, rest)
}
return rel
}
dir, rest = filepath.Dir(dir), filepath.Join(filepath.Base(dir), rest)
}
return rel
}
func (localsink *LocalSink) initialize(dir string, isIncremental bool) error {
localsink.Dir = dir
localsink.isIncremental = isIncremental
@@ -69,8 +98,17 @@ func (localsink *LocalSink) DeleteEntry(key string, isDirectory, deleteIncludeCh
if localsink.isMultiPartEntry(key) {
return nil
}
rel, ok := localsink.isInsideDir(key)
if !ok {
return nil
}
glog.V(4).Infof("Delete Entry key: %s", key)
if err := os.Remove(key); err != nil {
root, err := os.OpenRoot(localsink.Dir)
if err == nil {
err = root.Remove(localsink.resolveInsideDir(rel))
root.Close()
}
if err != nil {
glog.V(0).Infof("remove entry key %s: %s", key, err)
}
return nil
@@ -81,6 +119,10 @@ func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signa
if entry.IsDirectory || localsink.isMultiPartEntry(key) {
return nil
}
rel, ok := localsink.isInsideDir(key)
if !ok {
return nil
}
glog.V(4).Infof("Create Entry key: %s", key)
totalSize := filer.FileSize(entry)
@@ -89,35 +131,37 @@ func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signa
return fmt.Errorf("resolve chunks of %s: %w", key, err)
}
dir := filepath.Dir(key)
if _, err := os.Stat(dir); os.IsNotExist(err) {
glog.V(4).Infof("Create Directory key: %s", dir)
if err = os.MkdirAll(dir, 0755); err != nil {
return err
}
if err := os.MkdirAll(localsink.Dir, 0755); err != nil {
return err
}
root, err := os.OpenRoot(localsink.Dir)
if err != nil {
return err
}
defer root.Close()
if entry.IsDirectory {
return os.Mkdir(key, os.FileMode(entry.Attributes.FileMode))
rel = localsink.resolveInsideDir(rel)
dir := filepath.Dir(rel)
if err := root.MkdirAll(dir, 0755); err != nil {
return err
}
mode := os.FileMode(entry.Attributes.FileMode)
shortFileName := util.ToShortFileName(key)
shortFileName := util.ToShortFileName(rel)
// Write to a temp file in the same directory, then atomically rename
// on success. This prevents leaving a truncated/empty file if
// decryption or chunk copy fails.
tmpFile, err := os.CreateTemp(dir, ".seaweedfs-tmp-*")
tmpName := filepath.Join(dir, fmt.Sprintf(".seaweedfs-tmp-%d", rand.Uint64()))
tmpFile, err := root.OpenFile(tmpName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
if err != nil {
return err
}
tmpName := tmpFile.Name()
defer func() {
// Clean up temp file on any error (rename removes it on success)
if tmpFile != nil {
tmpFile.Close()
os.Remove(tmpName)
root.Remove(tmpName)
}
}()
@@ -151,8 +195,8 @@ func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signa
tmpFile = nil // prevent deferred cleanup
// Atomic rename into final destination
if err := os.Rename(tmpName, shortFileName); err != nil {
os.Remove(tmpName)
if err := root.Rename(tmpName, shortFileName); err != nil {
root.Remove(tmpName)
return err
}
@@ -164,6 +208,9 @@ func (localsink *LocalSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, ne
if localsink.isMultiPartEntry(key) {
return true, nil
}
if _, ok := localsink.isInsideDir(key); !ok {
return true, nil
}
glog.V(4).Infof("Update Entry key: %s", key)
// do delete and create
foundExistingEntry = util.FileExists(key)
@@ -0,0 +1,146 @@
package localsink
import (
"os"
"path/filepath"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
)
func TestLocalSinkStaysInsideDir(t *testing.T) {
root := t.TempDir()
dir := filepath.Join(root, "backup")
sink := &LocalSink{}
sink.initialize(dir, false)
sink.SetSourceFiler(&source.FilerSource{})
entry := func() *filer_pb.Entry {
return &filer_pb.Entry{
Name: "marker.txt",
Content: []byte("marker"),
Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6},
}
}
inside := filepath.Join(dir, "safe", "marker.txt")
if err := sink.CreateEntry(inside, entry(), nil); err != nil {
t.Fatalf("create inside: %v", err)
}
if _, err := os.Stat(inside); err != nil {
t.Fatalf("inside file missing: %v", err)
}
outside := filepath.Join(dir, "..", "escape", "marker.txt")
if err := sink.CreateEntry(outside, entry(), nil); err != nil {
t.Fatalf("create outside: %v", err)
}
if _, err := sink.UpdateEntry(outside, nil, "", entry(), false, nil); err != nil {
t.Fatalf("update outside: %v", err)
}
if _, err := os.Stat(filepath.Join(root, "escape")); !os.IsNotExist(err) {
t.Fatalf("sink wrote outside %s: %v", dir, err)
}
victim := filepath.Join(root, "victim.txt")
if err := os.WriteFile(victim, []byte("keep"), 0644); err != nil {
t.Fatal(err)
}
if err := sink.DeleteEntry(filepath.Join(dir, "..", "victim.txt"), false, false, nil); err != nil {
t.Fatalf("delete outside: %v", err)
}
if _, err := os.Stat(victim); err != nil {
t.Fatalf("sink deleted a file outside %s: %v", dir, err)
}
}
func TestLocalSinkDoesNotFollowSymlinksOutOfDir(t *testing.T) {
root := t.TempDir()
dir := filepath.Join(root, "backup")
outside := filepath.Join(root, "outside")
for _, d := range []string{dir, outside} {
if err := os.MkdirAll(d, 0755); err != nil {
t.Fatal(err)
}
}
if err := os.Symlink(outside, filepath.Join(dir, "link")); err != nil {
t.Skipf("symlink: %v", err)
}
sink := &LocalSink{}
sink.initialize(dir, false)
sink.SetSourceFiler(&source.FilerSource{})
entry := &filer_pb.Entry{
Name: "marker.txt",
Content: []byte("marker"),
Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6},
}
if err := sink.CreateEntry(filepath.Join(dir, "link", "marker.txt"), entry, nil); err == nil {
t.Fatalf("create through a symlink out of %s succeeded", dir)
}
if _, err := os.Stat(filepath.Join(outside, "marker.txt")); !os.IsNotExist(err) {
t.Fatalf("sink wrote through a symlink out of %s: %v", dir, err)
}
victim := filepath.Join(outside, "victim.txt")
if err := os.WriteFile(victim, []byte("keep"), 0644); err != nil {
t.Fatal(err)
}
if err := sink.DeleteEntry(filepath.Join(dir, "link", "victim.txt"), false, false, nil); err != nil {
t.Fatalf("delete through link: %v", err)
}
if _, err := os.Stat(victim); err != nil {
t.Fatalf("sink deleted through a symlink out of %s: %v", dir, err)
}
if err := os.Mkdir(filepath.Join(dir, "real"), 0755); err != nil {
t.Fatal(err)
}
if err := os.Symlink("real", filepath.Join(dir, "inner")); err != nil {
t.Fatal(err)
}
if err := sink.CreateEntry(filepath.Join(dir, "inner", "marker.txt"), entry, nil); err != nil {
t.Fatalf("create through a symlink inside %s: %v", dir, err)
}
if _, err := os.Stat(filepath.Join(dir, "real", "marker.txt")); err != nil {
t.Fatalf("inside symlink target missing: %v", err)
}
}
func TestLocalSinkFollowsAbsoluteSymlinksInsideDir(t *testing.T) {
dir := filepath.Join(t.TempDir(), "backup")
target := filepath.Join(dir, "real")
if err := os.MkdirAll(target, 0755); err != nil {
t.Fatal(err)
}
if err := os.Symlink(target, filepath.Join(dir, "abs")); err != nil {
t.Skipf("symlink: %v", err)
}
sink := &LocalSink{}
sink.initialize(dir, false)
sink.SetSourceFiler(&source.FilerSource{})
entry := &filer_pb.Entry{
Name: "marker.txt",
Content: []byte("marker"),
Attributes: &filer_pb.FuseAttributes{FileMode: 0644, FileSize: 6},
}
for _, name := range []string{"marker.txt", filepath.Join("sub", "marker.txt")} {
if err := sink.CreateEntry(filepath.Join(dir, "abs", name), entry, nil); err != nil {
t.Fatalf("create %s through an absolute symlink inside %s: %v", name, dir, err)
}
if _, err := os.Stat(filepath.Join(target, name)); err != nil {
t.Fatalf("absolute symlink target %s missing: %v", name, err)
}
if err := sink.DeleteEntry(filepath.Join(dir, "abs", name), false, false, nil); err != nil {
t.Fatalf("delete %s through an absolute symlink: %v", name, err)
}
if _, err := os.Stat(filepath.Join(target, name)); !os.IsNotExist(err) {
t.Fatalf("delete through an absolute symlink left %s: %v", name, err)
}
}
if _, err := os.Lstat(filepath.Join(dir, "abs")); err != nil {
t.Fatalf("symlink itself removed: %v", err)
}
}
+29 -33
View File
@@ -25,6 +25,7 @@ import (
"crypto/subtle"
"encoding/base64"
"encoding/hex"
"errors"
"io"
"net"
"net/http"
@@ -74,22 +75,19 @@ func streamHashRequestBody(r *http.Request, sizeLimit int64) (string, error) {
return emptySHA256, nil
}
limitedReader := io.LimitReader(r.Body, sizeLimit)
hasher := sha256.New()
var bodyBuffer bytes.Buffer
// Use io.Copy with an io.MultiWriter to hash and buffer the body simultaneously.
if _, err := io.Copy(io.MultiWriter(hasher, &bodyBuffer), limitedReader); err != nil {
body, err := readRequestBody(r, sizeLimit)
if err != nil {
return "", err
}
r.Body = io.NopCloser(bytes.NewReader(bodyBuffer.Bytes()))
r.Body = io.NopCloser(bytes.NewReader(body))
if bodyBuffer.Len() == 0 {
if len(body) == 0 {
return emptySHA256, nil
}
return hex.EncodeToString(hasher.Sum(nil)), nil
sum := sha256.Sum256(body)
return hex.EncodeToString(sum[:]), nil
}
// getContentSha256Cksum retrieves the "x-amz-content-sha256" header value.
@@ -374,35 +372,29 @@ func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCh
if trustedPeer {
forwardedPrefix = r.Header.Get("X-Forwarded-Prefix")
}
// 9. Candidate paths: the escaped path as sent, then the decoded path that
// an S3 client encodes once when signing; each with the forwarded prefix first.
signaturePaths := []string{pathForSignature}
if decodedPath, decodeErr := url.PathUnescape(pathForSignature); decodeErr == nil && decodedPath != pathForSignature {
signaturePaths = append(signaturePaths, decodedPath)
}
if forwardedPrefix != "" {
prefixedPaths := make([]string, 0, 2*len(signaturePaths))
for _, p := range signaturePaths {
prefixedPaths = append(prefixedPaths, buildPathWithForwardedPrefix(forwardedPrefix, p))
}
signaturePaths = append(prefixedPaths, signaturePaths...)
}
var matchedHost string
hostCandidates:
for i, hostCandidate := range extractHostHeaderCandidates(r, iam.externalHost) {
if i > 0 && !replaceSignedHostHeader(extractedSignedHeaders, hostCandidate) {
break
}
// 9. Verify with the X-Forwarded-Prefix path first
if forwardedPrefix != "" {
cleanedPath := buildPathWithForwardedPrefix(forwardedPrefix, pathForSignature)
calculatedSignature, errCode = verify(cleanedPath)
if errCode == s3err.ErrNone {
for _, p := range signaturePaths {
if calculatedSignature, errCode = verify(p); errCode == s3err.ErrNone {
matchedHost = hostCandidate
break
}
}
// 10. Verify with the original path
calculatedSignature, errCode = verify(pathForSignature)
if errCode == s3err.ErrNone {
matchedHost = hostCandidate
break
}
// 11. Retry with decoded path if signature used raw path encoding
if decodedPath, decodeErr := url.PathUnescape(pathForSignature); decodeErr == nil && decodedPath != pathForSignature {
calculatedSignature, errCode = verify(decodedPath)
if errCode == s3err.ErrNone {
matchedHost = hostCandidate
break
break hostCandidates
}
}
}
@@ -661,8 +653,12 @@ func extractV4AuthInfoFromHeader(r *http.Request) (*v4AuthInfo, s3err.ErrorCode)
hashedPayload := getContentSha256Cksum(r)
if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil {
sizeLimit := s3_constants.GetRequestBodyLimitFromContext(r, iamRequestBodyLimit)
var hashErr error
hashedPayload, hashErr = streamHashRequestBody(r, iamRequestBodyLimit)
hashedPayload, hashErr = streamHashRequestBody(r, sizeLimit)
if errors.Is(hashErr, errRequestBodyTooLarge) {
return nil, s3err.ErrEntityTooLarge
}
if hashErr != nil {
return nil, s3err.ErrInternalError
}
+33
View File
@@ -6,6 +6,7 @@ import (
"encoding/hex"
"fmt"
"net/http"
"net/url"
"slices"
"sync"
"testing"
@@ -735,3 +736,35 @@ func TestVerifyV4SignatureUsesOriginalRequestPath(t *testing.T) {
t.Fatalf("signed request rejected after path rewrite: %d", errCode)
}
}
// TestSignatureV4WithForwardedPrefixUnicodeKey guards against a signature
// mismatch when an object key contains non-ASCII characters and the request is
// received through a trusted reverse proxy that strips a path prefix. The
// client signs the single percent-encoded path including the prefix, so the
// server must decode the forwarded (already encoded) path before re-encoding it
// together with the prefix.
func TestSignatureV4WithForwardedPrefixUnicodeKey(t *testing.T) {
iam := newTestIAMWithTrustedProxy()
bucket := "test-bucket"
key := "たあいまd·.jpg"
// The proxy strips the X-Forwarded-Prefix and forwards the percent-encoded
// object path (the server sees the encoded form, not the raw Unicode key).
r, err := newTestRequest("GET", "https://example.com/"+bucket+"/"+url.PathEscape(key), 0, nil)
if err != nil {
t.Fatalf("failed to create test request: %v", err)
}
r.Header.Set("X-Forwarded-Prefix", "/s3")
r.Header.Set("Host", "example.com")
r.RemoteAddr = "10.0.0.1:1234"
// The client signs the full URL path including the prefix, using the single
// percent-encoded form of the Unicode key.
signV4WithPath(r, "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "/s3/"+bucket+"/"+key)
if _, _, errCode := iam.doesSignatureMatch(r); errCode != s3err.ErrNone {
t.Fatalf("signature verification failed for unicode key behind forwarded prefix: code %d", int(errCode))
}
}
+57 -28
View File
@@ -16,6 +16,8 @@ import (
"time"
"unicode/utf8"
"github.com/aws/aws-sdk-go/aws/credentials"
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
@@ -1820,18 +1822,10 @@ func TestIAMLargeBodySecurityLimit(t *testing.T) {
", SignedHeaders=content-type;host;x-amz-date, Signature=dummysignature"
req.Header.Set("Authorization", authHeader)
// The function should complete successfully but limit the body to 10 MiB
identity, _, errCode := iam.doesSignatureMatch(req)
// Should get signature mismatch (dummy signature) but not internal error
assert.Equal(t, s3err.ErrSignatureDoesNotMatch, errCode)
assert.Equal(t, s3err.ErrEntityTooLarge, errCode)
assert.Nil(t, identity)
// Verify the body was truncated to the limit (10 MiB)
bodyBytes, err := io.ReadAll(req.Body)
assert.NoError(t, err)
assert.Equal(t, 10*(1<<20), len(bodyBytes)) // Should be exactly 10 MiB
assert.Equal(t, strings.Repeat("A", 10*(1<<20)), string(bodyBytes)) // All As, but truncated
}
// Test the streaming hash implementation directly
@@ -1922,30 +1916,65 @@ func TestStreamingVsNonStreamingConsistency(t *testing.T) {
}
}
// Test streaming with size limit enforcement
func TestStreamingWithSizeLimit(t *testing.T) {
// Create a payload larger than the limit
largePayload := strings.Repeat("A", 11*(1<<20)) // 11 MiB
req, err := http.NewRequest("POST", "http://localhost:8111/", strings.NewReader(largePayload))
exact := strings.Repeat("A", 1<<20)
req, err := http.NewRequest("POST", "http://localhost:8111/", strings.NewReader(exact))
assert.NoError(t, err)
// Stream with the limit
hash, err := streamHashRequestBody(req, iamRequestBodyLimit)
hash, err := streamHashRequestBody(req, 1<<20)
assert.NoError(t, err)
assert.Equal(t, getSHA256Hash([]byte(exact)), hash)
// Verify the hash is computed for the truncated content (10 MiB)
truncatedPayload := strings.Repeat("A", 10*(1<<20))
expectedHash := sha256.Sum256([]byte(truncatedPayload))
expectedHashStr := hex.EncodeToString(expectedHash[:])
assert.Equal(t, expectedHashStr, hash)
// Verify the body was truncated
bodyBytes, err := io.ReadAll(req.Body)
req, err = http.NewRequest("POST", "http://localhost:8111/", strings.NewReader(exact+"A"))
assert.NoError(t, err)
assert.Equal(t, 10*(1<<20), len(bodyBytes))
assert.Equal(t, truncatedPayload, string(bodyBytes))
_, err = streamHashRequestBody(req, 1<<20)
assert.ErrorIs(t, err, errRequestBodyTooLarge)
}
func TestNonS3BodyHashLimitFollowsRoute(t *testing.T) {
iam := &IdentityAccessManagement{}
assert.NoError(t, iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
Identities: []*iam_pb.Identity{{
Name: "vec",
Credentials: []*iam_pb.Credential{{AccessKey: "vec_key", SecretKey: "vec_secret"}},
Actions: []string{"Admin"},
}},
}))
signer := v4.NewSigner(credentials.NewStaticCredentials("vec_key", "vec_secret", ""))
vectorsRoute := int64(s3_constants.S3VectorsMaxRequestBodyBytes)
for _, tc := range []struct {
service string
routeLimit int64
size int
want s3err.ErrorCode
}{
{"s3vectors", vectorsRoute, 20 << 20, s3err.ErrNone},
{"s3vectors", vectorsRoute, s3_constants.S3VectorsMaxRequestBodyBytes, s3err.ErrNone},
{"s3vectors", vectorsRoute, s3_constants.S3VectorsMaxRequestBodyBytes + 1, s3err.ErrEntityTooLarge},
// An s3vectors scope alone, on another route, keeps the 10 MiB limit.
{"s3vectors", 0, iamRequestBodyLimit + 1, s3err.ErrEntityTooLarge},
{"iam", 0, iamRequestBodyLimit + 1, s3err.ErrEntityTooLarge},
} {
t.Run(fmt.Sprintf("%s_route%d_%d", tc.service, tc.routeLimit, tc.size), func(t *testing.T) {
body := strings.Repeat("v", tc.size)
req, err := http.NewRequest("POST", "http://localhost:8333/PutVectors", strings.NewReader(body))
assert.NoError(t, err)
if tc.routeLimit > 0 {
req = req.WithContext(s3_constants.SetRequestBodyLimitInContext(req.Context(), tc.routeLimit))
}
_, err = signer.Sign(req, strings.NewReader(body), tc.service, "us-east-1", time.Now())
assert.NoError(t, err)
identity, _, errCode := iam.doesSignatureMatch(req)
assert.Equal(t, tc.want, errCode)
if tc.want == s3err.ErrNone {
assert.Equal(t, "vec", identity.Name)
got, err := io.ReadAll(req.Body)
assert.NoError(t, err)
assert.Equal(t, len(body), len(got))
}
})
}
}
// Benchmark streaming vs non-streaming memory usage
+4
View File
@@ -282,6 +282,10 @@ func (s *Server) authorizeCommit(ctx context.Context, bucketARN string, namespac
Name: tableName,
}, identityName)
})
if err == nil {
bucketName, _ := s3tables.ParseBucketNameFromARN(bucketARN)
err = s.tablesManager.CheckVectorNamespaceWrite(ctx, s.filerClient, bucketName, namespace)
}
if err == nil {
return nil
}
+9
View File
@@ -308,6 +308,12 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
writeManagerError(w, authErr)
return
}
// Registration confines the location to the table's namespace, but both
// branches write, and stage-create vends credentials, before it runs.
if err := s3tables.ValidateMetadataLocation(metadataLocation, bucketName, flattenNamespacePath(namespace)); err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", err.Error())
return
}
// Stage-create persists metadata in the internal staged area and skips S3Tables registration.
if req.StageCreate {
@@ -786,6 +792,9 @@ func (s *Server) mayUpdateTable(r *http.Request, bucketName string, namespace []
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return s.tablesManager.AuthorizeUpdateTable(r.Context(), s3tables.NewManagerClient(client), updateReq, identityName)
})
if err == nil {
err = s.tablesManager.CheckVectorNamespaceWrite(r.Context(), s.filerClient, bucketName, namespace)
}
if err != nil {
glog.V(1).Infof("Iceberg: UpdateTable check for %s/%s failed, vending read-only: %v", bucketName, tableName, err)
}
+130
View File
@@ -0,0 +1,130 @@
package iceberg
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
)
type openAuthenticator struct{}
func (openAuthenticator) AuthenticateRequest(*http.Request) (string, interface{}, s3err.ErrorCode) {
return s3tables.DefaultAccountID, nil, s3err.ErrNone
}
func (openAuthenticator) DefaultAllow() bool { return true }
// An index in a vector bucket is a catalog table the Iceberg catalog can load,
// but a caller holding every s3tables grant gets neither write credentials to
// it nor a commit into it.
func TestVectorIndexIsReadOnlyOverIceberg(t *testing.T) {
filer := s3tablestest.Start(t)
vendor := &stubVendor{credentials: &VendedCredentials{AccessKeyID: "ASIAEXAMPLE"}}
s := NewServer(s3tables.NewManagerClient(filer.Client), openAuthenticator{})
s.SetCredentialVendor(vendor)
s.s3Endpoint = "http://127.0.0.1:8333"
router := mux.NewRouter().SkipClean(true)
s.RegisterRoutes(router)
admin := s3tables.NewManager()
admin.SetTrusted(true)
client := s3tables.NewManagerClient(filer.Client)
arn, err := s3tables.BuildBucketARN(s3tables.DefaultRegion, s3tables.DefaultAccountID, "vec")
if err != nil {
t.Fatal(err)
}
metadata, _ := json.Marshal(map[string]any{"dimension": 4, "distanceMetric": "cosine"})
for _, op := range []struct {
name string
req interface{}
}{
{"CreateTableBucket", &s3tables.CreateTableBucketRequest{Name: "vec", Format: s3tables.FormatLance}},
{"CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}}},
{"CreateTable", &s3tables.CreateTableRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace},
Name: "idx", Format: s3tables.FormatLance, Metadata: &s3tables.TableMetadata{FullMetadata: metadata}}},
} {
if err := admin.Execute(t.Context(), client, op.name, op.req, nil, ""); err != nil {
t.Fatalf("%s: %v", op.name, err)
}
}
req := httptest.NewRequest(http.MethodGet, "/v1/vec/namespaces/vector/tables/idx", nil)
req.Header.Set(accessDelegationHeader, "vended-credentials")
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("load = %d (%s)", rec.Code, rec.Body.String())
}
if vendor.gotPrefix != "vector/idx" || !vendor.gotReadOnly {
t.Errorf("vended prefix %q read-only %v, want vector/idx read-only", vendor.gotPrefix, vendor.gotReadOnly)
}
reqErr := s.authorizeCommit(t.Context(), arn, []string{s3tables.VectorIndexNamespace}, "idx", s3tables.DefaultAccountID)
if reqErr == nil || reqErr.status != http.StatusForbidden {
t.Errorf("commit authorization = %+v, want 403", reqErr)
}
}
// Stage-create registers nothing, so the catalog's own location and vector
// namespace checks never see it. It must not hand out write credentials to an
// index, whether the staged table names the index's location from another
// namespace or sits in the vector namespace itself.
func TestStageCreateCannotReachVectorIndex(t *testing.T) {
filer := s3tablestest.Start(t)
s := NewServer(s3tables.NewManagerClient(filer.Client), openAuthenticator{})
s.s3Endpoint = "http://127.0.0.1:8333"
router := mux.NewRouter().SkipClean(true)
s.RegisterRoutes(router)
admin := s3tables.NewManager()
admin.SetTrusted(true)
client := s3tables.NewManagerClient(filer.Client)
arn, err := s3tables.BuildBucketARN(s3tables.DefaultRegion, s3tables.DefaultAccountID, "vec")
if err != nil {
t.Fatal(err)
}
for _, op := range []struct {
name string
req interface{}
}{
{"CreateTableBucket", &s3tables.CreateTableBucketRequest{Name: "vec", Format: s3tables.FormatLance}},
{"CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace}}},
{"CreateNamespace", &s3tables.CreateNamespaceRequest{TableBucketARN: arn, Namespace: []string{"other"}}},
{"CreateTable", &s3tables.CreateTableRequest{TableBucketARN: arn, Namespace: []string{s3tables.VectorIndexNamespace},
Name: "idx", Format: s3tables.FormatLance}},
} {
if err := admin.Execute(t.Context(), client, op.name, op.req, nil, ""); err != nil {
t.Fatalf("%s: %v", op.name, err)
}
}
for name, tc := range map[string]struct {
namespace, body string
status int
}{
"index location from another namespace": {"other", `{"name":"x","stage-create":true,"location":"s3://vec/vector/idx"}`, http.StatusBadRequest},
"vector namespace": {s3tables.VectorIndexNamespace, `{"name":"y","stage-create":true}`, http.StatusForbidden},
} {
t.Run(name, func(t *testing.T) {
vendor := &stubVendor{credentials: &VendedCredentials{AccessKeyID: "ASIAEXAMPLE"}}
s.SetCredentialVendor(vendor)
req := httptest.NewRequest(http.MethodPost, "/v1/vec/namespaces/"+tc.namespace+"/tables", strings.NewReader(tc.body))
req.Header.Set(accessDelegationHeader, "vended-credentials")
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != tc.status {
t.Errorf("stage-create = %d (%s), want %d", rec.Code, rec.Body.String(), tc.status)
}
if vendor.gotPrefix != "" && !vendor.gotReadOnly {
t.Errorf("vended write credentials to %q", vendor.gotPrefix)
}
})
}
}
+94
View File
@@ -0,0 +1,94 @@
package lance
import (
"encoding/hex"
"net/http"
"net/http/httptest"
"testing"
"time"
jwt "github.com/golang-jwt/jwt/v5"
"github.com/seaweedfs/seaweedfs/weed/security"
)
func signAdminClaims(t *testing.T, key security.SigningKey, claims security.SeaweedFilerAdminClaims) string {
t.Helper()
token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(key))
if err != nil {
t.Fatalf("sign: %v", err)
}
return token
}
func trustedSeenByHandler(t *testing.T, s *Server, token string) bool {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil)
if token != "" {
req.Header.Set(clusterTokenHeader, token)
}
trusted := false
rec := httptest.NewRecorder()
s.Auth(func(w http.ResponseWriter, r *http.Request) {
trusted = isTrustedCaller(r.Context())
w.WriteHeader(http.StatusOK)
})(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d (%s), want 200", rec.Code, rec.Body.String())
}
return trusted
}
func TestClusterTokenMarksTheCallerTrusted(t *testing.T) {
key := security.SigningKey("cluster-key")
s := NewServer(nil, openAuthenticator{})
s.SetSigningKey(func() security.SigningKey { return key })
derived := clusterTokenKey(key)
expired := security.SeaweedFilerAdminClaims{}
expired.ExpiresAt = jwt.NewNumericDate(time.Now().Add(-time.Minute))
for _, tc := range []struct {
name string
token string
want bool
}{
{"no token", "", false},
{"signed with the derived key", string(security.GenJwtForFilerAdmin(derived, 60)), true},
{"signed with the cluster key itself", string(security.GenJwtForFilerAdmin(key, 60)), false},
{"signed with another key", string(security.GenJwtForFilerAdmin(clusterTokenKey(security.SigningKey("other-key")), 60)), false},
{"expired", signAdminClaims(t, derived, expired), false},
{"session token", signAdminClaims(t, derived, security.SeaweedFilerAdminClaims{SessionId: "session"}), false},
{"garbage", "not-a-token", false},
} {
t.Run(tc.name, func(t *testing.T) {
if got := trustedSeenByHandler(t, s, tc.token); got != tc.want {
t.Errorf("trusted = %v, want %v", got, tc.want)
}
})
}
token := string(security.GenJwtForFilerAdmin(derived, 60))
key = security.SigningKey("rotated-key")
if trustedSeenByHandler(t, s, token) {
t.Error("a token from before a key rotation is still trusted")
}
if !trustedSeenByHandler(t, s, string(security.GenJwtForFilerAdmin(clusterTokenKey(key), 60))) {
t.Error("a token signed after a key rotation is not trusted")
}
key = nil
if trustedSeenByHandler(t, s, token) {
t.Error("a server with no cluster key trusted a caller")
}
s.SetSigningKey(nil)
if trustedSeenByHandler(t, s, token) {
t.Error("a server with no key source trusted a caller")
}
}
// The Rust catalog client derives the same key; its tests pin this value too.
func TestClusterTokenKeyDerivation(t *testing.T) {
got := hex.EncodeToString(clusterTokenKey(security.SigningKey("cluster-key")))
if want := "b38c0b66da41621f2a9b6de68e4239ddd0d2dd7b4ede01aa91bd34a17edc1021"; got != want {
t.Errorf("clusterTokenKey = %s, want %s", got, want)
}
}
+31 -2
View File
@@ -49,8 +49,9 @@ func newTestServerWithOAuth() *Server {
}
type mockS3Authenticator struct {
called bool
errCode s3err.ErrorCode
called bool
errCode s3err.ErrorCode
authDisabled bool
}
func (m *mockS3Authenticator) AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode) {
@@ -63,6 +64,8 @@ func (m *mockS3Authenticator) AuthenticateRequest(r *http.Request) (string, inte
func (m *mockS3Authenticator) DefaultAllow() bool { return false }
func (m *mockS3Authenticator) AuthEnabled() bool { return !m.authDisabled }
func mintTestToken(t *testing.T, accessKey, secret string, issuedAt, expiresAt time.Time) string {
t.Helper()
key := deriveSigningKey(accessKey, secret)
@@ -316,6 +319,32 @@ func TestAuthApiKeyInvalid(t *testing.T) {
}
}
// Without identities the S3 port takes any signature, so the catalog takes any
// key or token and leaves the caller to the S3 authenticator.
func TestAuthTakesAnyKeyWhenS3AuthIsOff(t *testing.T) {
s := newTestServerWithOAuth()
s.authenticator = &mockS3Authenticator{authDisabled: true}
for header, value := range map[string]string{
"x-api-key": "unknown:key",
"Authorization": "Bearer not-a-token",
} {
var gotIdentity string
handler := s.Auth(func(w http.ResponseWriter, r *http.Request) {
gotIdentity = s3_constants.GetIdentityNameFromContext(r)
w.WriteHeader(http.StatusOK)
})
req := httptest.NewRequest(http.MethodGet, "/v1/namespace/$/list", nil)
req.Header.Set(header, value)
rec := httptest.NewRecorder()
handler(rec, req)
if rec.Code != http.StatusOK || gotIdentity != "s3user" {
t.Fatalf("%s: status = %d, identity = %q, want 200 as s3user", header, rec.Code, gotIdentity)
}
}
}
func TestAuthNoBearerStillUsesS3Authenticator(t *testing.T) {
s := newTestServerWithOAuth()
auth := &mockS3Authenticator{errCode: s3err.ErrNone}
+34 -15
View File
@@ -135,24 +135,35 @@ func (s *Server) handleDeclareTable(w http.ResponseWriter, r *http.Request) {
return
}
if err := s.createTable(r, bucket, ns, name, location); err != nil {
_, lookupErr := s.loadLanceTable(r, bucket, ns, name)
location, err = s.createTable(r, bucket, ns, name, location)
if err != nil {
writeStorageError(w, r, err)
return
}
if err := s.writeMarker(r, location, reservedMarker); err != nil {
// The catalog entry is the authority; the marker only mirrors it for
// clients that read the storage prefix directly.
glog.V(1).Infof("lance: could not write %s for %s: %v", reservedMarker, location, err)
// A declare that creates the table may write it: a Lance commit is a direct
// S3 write that never revisits the catalog, so the create authorization is
// the write evidence. One that only finds the table answers like describe.
var writable bool
if isNotFound(lookupErr) {
writable = s.mayWriteNamespace(r, bucket, ns)
} else {
writable = s.mayUpdateTable(r, bucket, ns, name)
}
// Declaring a name that was deregistered brings it back, the same way
// registering it does.
if err := s.removeMarker(r, location, deregisteredMarker); err != nil {
glog.V(2).Infof("lance: could not clear %s for %s: %v", deregisteredMarker, location, err)
if writable {
if err := s.writeMarker(r, location, reservedMarker); err != nil {
// The catalog entry is the authority; the marker only mirrors it for
// clients that read the storage prefix directly.
glog.V(1).Infof("lance: could not write %s for %s: %v", reservedMarker, location, err)
}
// Declaring a name that was deregistered brings it back, the same way
// registering it does.
if err := s.removeMarker(r, location, deregisteredMarker); err != nil {
glog.V(2).Infof("lance: could not clear %s for %s: %v", deregisteredMarker, location, err)
}
}
// A Lance commit is a direct S3 write that never revisits the catalog, so
// the create authorization above is the write evidence for the dataset.
options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials), func() bool { return true })
options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials), func() bool { return writable })
if err != nil {
writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
return
@@ -353,7 +364,7 @@ func (s *Server) handleRegisterTable(w http.ResponseWriter, r *http.Request) {
}
}
default:
if err := s.createTable(r, bucket, ns, name, location); err != nil {
if location, err = s.createTable(r, bucket, ns, name, location); err != nil {
writeStorageError(w, r, err)
return
}
@@ -545,7 +556,9 @@ func (s *Server) loadLanceTable(r *http.Request, bucket string, ns []string, nam
return &lanceTable{location: location, versionToken: resp.VersionToken}, nil
}
func (s *Server) createTable(r *http.Request, bucket string, ns []string, name, location string) error {
// createTable returns the location the catalog holds for the table. When the
// name already exists that is its own location, not the one asked for.
func (s *Server) createTable(r *http.Request, bucket string, ns []string, name, location string) (string, error) {
req := &s3tables.CreateTableRequest{
TableBucketARN: bucketARN(bucket),
Namespace: ns,
@@ -556,7 +569,13 @@ func (s *Server) createTable(r *http.Request, bucket string, ns []string, name,
MetadataLocation: location,
}
var resp s3tables.CreateTableResponse
return s.execute(r, "CreateTable", req, &resp)
if err := s.execute(r, "CreateTable", req, &resp); err != nil {
return "", err
}
if resp.MetadataLocation == "" {
return tableLocation(bucket, ns, name), nil
}
return resp.MetadataLocation, nil
}
// repointTable moves an existing entry to another dataset location without
+75
View File
@@ -24,6 +24,14 @@ func (openAuthenticator) AuthenticateRequest(*http.Request) (string, interface{}
func (openAuthenticator) DefaultAllow() bool { return true }
func (openAuthenticator) AuthEnabled() bool { return false }
// signedAuthenticator admits the same caller on a gateway that checks
// credentials.
type signedAuthenticator struct{ openAuthenticator }
func (signedAuthenticator) AuthEnabled() bool { return true }
type testHarness struct {
router *mux.Router
filer *s3tablestest.MemFiler
@@ -134,6 +142,28 @@ func decode[T any](t *testing.T, recorder *httptest.ResponseRecorder) T {
return out
}
// While the S3 port takes unsigned requests the catalog says so, and a client
// with no credentials of its own still reaches the data. Once auth is on it
// has to sign again.
func TestStorageOptionsSkipSignatureOnlyWhileS3AuthIsOff(t *testing.T) {
h := newTestHarness(t)
h.createBucket(t, "open", s3tables.FormatLance)
h.mustDo(t, http.MethodPost, "/v1/namespace/open$ns/create", `{}`, http.StatusOK)
declared := decode[DeclareTableResponse](t,
h.mustDo(t, http.MethodPost, "/v1/table/open$ns$t/declare", `{}`, http.StatusOK))
if declared.StorageOptions["aws_skip_signature"] != "true" {
t.Fatalf("auth off: storage options = %v, want aws_skip_signature", declared.StorageOptions)
}
h.server.authenticator = signedAuthenticator{}
described := decode[DescribeTableResponse](t,
h.mustDo(t, http.MethodPost, "/v1/table/open$ns$t/describe", `{}`, http.StatusOK))
if _, ok := described.StorageOptions["aws_skip_signature"]; ok {
t.Fatalf("auth on: storage options = %v, want no aws_skip_signature", described.StorageOptions)
}
}
// The lifecycle a Lance client drives: create the namespace, declare the table
// before any data exists, resolve it to a location, then deregister and bring it
// back by registering the same location.
@@ -415,3 +445,48 @@ func TestDataPlaneIsUnsupported(t *testing.T) {
t.Fatalf("error code = %d, want %d", got.Code, codeUnsupported)
}
}
// createOnlyAuthenticator signs in an identity that may create and read tables
// but not update them.
type createOnlyAuthenticator struct{}
func (createOnlyAuthenticator) AuthenticateRequest(*http.Request) (string, interface{}, s3err.ErrorCode) {
return "creator", &struct {
Name string
Actions []string
}{Name: "creator", Actions: []string{"s3tables:CreateTable", "s3tables:GetTable"}}, s3err.ErrNone
}
func (createOnlyAuthenticator) DefaultAllow() bool { return false }
func (createOnlyAuthenticator) AuthEnabled() bool { return true }
// The right to create a name is the write evidence for a table the declare
// creates, but not for one it only finds.
func TestDeclareVendsWriteOnlyForATableItCreates(t *testing.T) {
h := newTestHarness(t)
h.createBucket(t, "lnc", s3tables.FormatLance)
h.mustDo(t, http.MethodPost, "/v1/namespace/lnc$ns/create", `{}`, http.StatusOK)
h.mustDo(t, http.MethodPost, "/v1/table/lnc$ns$theirs/declare", `{}`, http.StatusOK)
h.mustDo(t, http.MethodPost, "/v1/table/lnc$ns$theirs/deregister", `{}`, http.StatusOK)
server := NewServer(s3tables.NewManagerClient(h.filer.Client), createOnlyAuthenticator{})
server.SetCredentialVendor(recordingVendor{})
router := mux.NewRouter().SkipClean(true)
server.RegisterRoutes(router)
creator := &testHarness{router: router, filer: h.filer, admin: h.admin, server: server}
for _, tc := range []struct{ name, want string }{
{"theirs", "read-only"},
{"mine", "read-write"},
} {
t.Run(tc.name, func(t *testing.T) {
if got := creator.vendedKey(t, "/v1/table/lnc$ns$"+tc.name+"/declare", ""); got != tc.want {
t.Errorf("vended %q, want %q", got, tc.want)
}
})
}
if h.filer.Get("/buckets/lnc/ns/theirs", deregisteredMarker) == nil {
t.Error("a declare that may not write the table brought it back")
}
}
+68 -2
View File
@@ -2,6 +2,8 @@ package lance
import (
"context"
"crypto/hmac"
"crypto/sha256"
"net/http"
"strings"
"time"
@@ -12,6 +14,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
"github.com/seaweedfs/seaweedfs/weed/security"
)
// FilerClient provides access to the filer for storage operations.
@@ -22,6 +25,8 @@ type FilerClient interface {
type S3Authenticator interface {
AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode)
DefaultAllow() bool
// AuthEnabled reports whether the S3 server checks credentials at all.
AuthEnabled() bool
}
// CredentialValidator validates S3 access key / secret key pairs
@@ -61,6 +66,7 @@ type Server struct {
credentialVendor CredentialVendor
s3Endpoint string
s3Region string
signingKey func() security.SigningKey
}
// NewServer creates a Lance namespace server over the given filer.
@@ -89,6 +95,13 @@ func (s *Server) SetCredentialVendor(vendor CredentialVendor) {
s.credentialVendor = vendor
}
// SetSigningKey sets where to read the cluster key (jwt.filer_signing.key)
// that a trusted caller's clusterTokenHeader is checked against. It is read on
// every check, so a reloaded key takes effect at once.
func (s *Server) SetSigningKey(signingKey func() security.SigningKey) {
s.signingKey = signingKey
}
// SetS3Endpoint configures the S3 endpoint advertised in storage_options so a
// client can reach the dataset without separately discovering the S3 address.
func (s *Server) SetS3Endpoint(endpoint string) {
@@ -166,17 +179,27 @@ func loggingMiddleware(next http.Handler) http.Handler {
})
}
// authEnabled is false when the S3 server has no identities and takes any
// signature. The catalog then takes any key too, and leaves the caller to the
// S3 authenticator.
func (s *Server) authEnabled() bool {
return s.authenticator == nil || s.authenticator.AuthEnabled()
}
// Auth authenticates the caller and puts the identity in the request context.
// Lance clients authenticate the catalog with a Bearer token or an x-api-key
// header; the S3 authenticator stays for callers that can SigV4-sign.
func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if s.hasClusterToken(r) {
r = r.WithContext(withTrustedCaller(r.Context()))
}
// A request carrying a Bearer token is a Lance REST client. An invalid
// or expired token gets 401 immediately; falling through to the S3
// authenticator would parse the header as SigV4 and fail with a
// different error that clients cannot act on.
// The auth scheme is case-insensitive (RFC 7235).
if strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") {
if s.authEnabled() && strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") {
if identityName, identity, accessKey, secretKey, ok := s.authenticateBearerKey(r); ok {
ctx := r.Context()
ctx = s3_constants.SetIdentityNameInContext(ctx, identityName)
@@ -194,7 +217,7 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
return
}
if apiKey := r.Header.Get("x-api-key"); apiKey != "" {
if apiKey := r.Header.Get("x-api-key"); apiKey != "" && s.authEnabled() {
if identityName, identity, ok := s.authenticateApiKey(apiKey); ok {
accessKey, secretKey, _ := strings.Cut(apiKey, ":")
ctx := r.Context()
@@ -250,3 +273,46 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
handler(w, r)
}
}
// clusterTokenHeader carries a token signed with clusterTokenKey. The vector
// engine and the Lance maintenance worker send it next to their identity's
// credentials, and only they may write index data in a vector bucket.
const clusterTokenHeader = "X-Seaweedfs-Cluster-Token"
// clusterTokenKey derives the key cluster tokens are signed with from the
// cluster key, so a token captured on its way here does not authorize the
// admin calls that take the cluster key itself.
func clusterTokenKey(key security.SigningKey) security.SigningKey {
mac := hmac.New(sha256.New, key)
mac.Write([]byte("seaweedfs-lance-cluster-token"))
return mac.Sum(nil)
}
type trustedCallerKey struct{}
func withTrustedCaller(ctx context.Context) context.Context {
return context.WithValue(ctx, trustedCallerKey{}, true)
}
func isTrustedCaller(ctx context.Context) bool {
trusted, _ := ctx.Value(trustedCallerKey{}).(bool)
return trusted
}
// hasClusterToken reports whether the request proves it holds the cluster key.
// Without a configured key no caller can.
func (s *Server) hasClusterToken(r *http.Request) bool {
token := r.Header.Get(clusterTokenHeader)
if token == "" || s.signingKey == nil {
return false
}
key := s.signingKey()
if len(key) == 0 {
return false
}
if security.IsFilerAdminToken(clusterTokenKey(key), security.EncodedJwt(token)) {
return true
}
glog.V(1).Infof("lance: %s from %s is not signed with a key derived from jwt.filer_signing.key", clusterTokenHeader, r.RemoteAddr)
return false
}
+22
View File
@@ -149,6 +149,11 @@ func (s *Server) storageOptions(r *http.Request, bucket, location string, vend b
if s.s3Region != "" {
options["aws_region"] = s.s3Region
}
if !s.authEnabled() {
// The S3 port takes unsigned requests, and a client with no
// credentials of its own would otherwise fail looking for some.
options["aws_skip_signature"] = "true"
}
if !vend || s.credentialVendor == nil {
return options, nil
@@ -177,6 +182,9 @@ func (s *Server) storageOptions(r *http.Request, bucket, location string, vend b
// credentials would cover, the same gate the Iceberg catalog applies so a
// read-only caller never gets write-capable storage_options.
func (s *Server) mayUpdateTable(r *http.Request, bucket string, ns []string, name string) bool {
if !s.mayWriteNamespace(r, bucket, ns) {
return false
}
req := &s3tables.UpdateTableRequest{
TableBucketARN: bucketARN(bucket),
Namespace: ns,
@@ -192,6 +200,20 @@ func (s *Server) mayUpdateTable(r *http.Request, bucket string, ns []string, nam
return err == nil
}
// mayWriteNamespace keeps the vector namespace of a vector bucket out of reach
// of s3tables grants: its index data is written only by trusted callers, and
// the vector bucket policy gates everyone else through S3 Vectors.
func (s *Server) mayWriteNamespace(r *http.Request, bucket string, ns []string) bool {
if isTrustedCaller(r.Context()) {
return true
}
err := s.tablesManager.CheckVectorNamespaceWrite(r.Context(), s.filerClient, bucket, ns)
if err != nil {
glog.V(1).Infof("lance: %s/%s is read-only to this caller: %v", bucket, strings.Join(ns, "."), err)
}
return err == nil
}
// locationPrefix strips the s3://bucket/ part of a location, leaving the key
// prefix a credential is scoped to.
func locationPrefix(location string) string {
+88
View File
@@ -1,10 +1,14 @@
package lance
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
"github.com/seaweedfs/seaweedfs/weed/security"
)
// seedVectorIndex creates bucket/vector/idx through the trusted path, the way
@@ -75,3 +79,87 @@ func TestVectorNamespaceOfUndeclaredBucketIsWritableOverLance(t *testing.T) {
h.mustDo(t, http.MethodPost, idx+"/register", `{"location":"s3://old/vector/idx"}`, http.StatusOK)
h.mustDo(t, http.MethodPost, idx+"/drop", `{}`, http.StatusOK)
}
// recordingVendor vends a credential whose key says whether it may write and
// whose secret is the prefix it covers.
type recordingVendor struct{}
func (recordingVendor) VendTableCredentials(_ context.Context, _, _, prefix string, readOnly bool) (*VendedCredentials, error) {
if readOnly {
return &VendedCredentials{AccessKeyID: "read-only", SecretAccessKey: prefix}, nil
}
return &VendedCredentials{AccessKeyID: "read-write", SecretAccessKey: prefix}, nil
}
func (h *testHarness) vendedKey(t *testing.T, target, token string) string {
t.Helper()
req := httptest.NewRequest(http.MethodPost, target, strings.NewReader(`{"vend_credentials":true}`))
if token != "" {
req.Header.Set(clusterTokenHeader, token)
}
rec := httptest.NewRecorder()
h.router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("POST %s = %d (%s)", target, rec.Code, rec.Body.String())
}
return decode[DescribeTableResponse](t, rec).StorageOptions["aws_access_key_id"]
}
// A caller holding every s3tables grant still gets read-only credentials to a
// vector bucket's index data; only the cluster's own components can write it.
func TestVectorNamespaceVendsWriteCredentialsOnlyToTrustedCallers(t *testing.T) {
h := newTestHarness(t)
key := security.SigningKey("cluster-key")
h.server.SetSigningKey(func() security.SigningKey { return key })
h.server.SetCredentialVendor(recordingVendor{})
h.seedVectorIndex(t, "vec", s3tables.FormatLance)
h.mustDo(t, http.MethodPost, "/v1/table/vec$other$tbl/declare", `{}`, http.StatusOK)
h.seedVectorIndex(t, "old", "")
h.vendedKey(t, "/v1/table/vec$vector$idx/declare", "")
if h.filer.Get("/buckets/vec/vector/idx", reservedMarker) != nil {
t.Error("an untrusted declare wrote into the index")
}
trusted := string(security.GenJwtForFilerAdmin(clusterTokenKey(key), 60))
forged := string(security.GenJwtForFilerAdmin(key, 60))
for _, tc := range []struct {
name, target, token, want string
}{
{"describe the index", "/v1/table/vec$vector$idx/describe", "", "read-only"},
{"declare the index", "/v1/table/vec$vector$idx/declare", "", "read-only"},
{"describe with an admin token", "/v1/table/vec$vector$idx/describe", forged, "read-only"},
{"trusted describe", "/v1/table/vec$vector$idx/describe", trusted, "read-write"},
{"trusted declare", "/v1/table/vec$vector$idx/declare", trusted, "read-write"},
{"another namespace", "/v1/table/vec$other$tbl/describe", "", "read-write"},
{"bucket from before formats", "/v1/table/old$vector$idx/describe", "", "read-write"},
} {
t.Run(tc.name, func(t *testing.T) {
if got := h.vendedKey(t, tc.target, tc.token); got != tc.want {
t.Errorf("vended %q, want %q", got, tc.want)
}
})
}
}
// Declaring a name that already exists answers for that table, so naming the
// index's location on another table gets no credentials to the index.
func TestDeclareOfAnExistingTableKeepsItsLocation(t *testing.T) {
h := newTestHarness(t)
h.server.SetCredentialVendor(recordingVendor{})
h.seedVectorIndex(t, "vec", s3tables.FormatLance)
h.mustDo(t, http.MethodPost, "/v1/table/vec$other$tbl/declare", `{}`, http.StatusOK)
rec := h.mustDo(t, http.MethodPost, "/v1/table/vec$other$tbl/declare",
`{"location":"s3://vec/vector/idx","vend_credentials":true}`, http.StatusOK)
resp := decode[DeclareTableResponse](t, rec)
if resp.Location != "s3://vec/other/tbl" {
t.Errorf("location = %q, want the table's own s3://vec/other/tbl", resp.Location)
}
if got := resp.StorageOptions["aws_secret_access_key"]; got != "other/tbl" {
t.Errorf("credentials cover %q, want other/tbl", got)
}
if h.filer.Get("/buckets/vec/vector/idx", reservedMarker) != nil {
t.Error("the declare wrote into the index")
}
}
+21 -1
View File
@@ -213,12 +213,28 @@ func (s3a *S3ApiServer) getEncryptionConfiguration(bucket string) (*s3_pb.Encryp
}
if metadata.Encryption == nil {
return nil, s3err.ErrNoSuchBucketEncryptionConfiguration
return s3a.declaredEncryption(bucket)
}
return metadata.Encryption, s3err.ErrNone
}
// declaredEncryption is the encryption a table bucket's record declares. It
// applies while the bucket has no S3 default encryption of its own.
func (s3a *S3ApiServer) declaredEncryption(bucket string) (*s3_pb.EncryptionConfiguration, s3err.ErrorCode) {
if s3a.bucketConfigCache == nil {
return nil, s3err.ErrNoSuchBucketEncryptionConfiguration
}
config, errCode := s3a.getBucketConfig(bucket)
if errCode != s3err.ErrNone {
return nil, errCode
}
if config.DeclaredEncryption == nil {
return nil, s3err.ErrNoSuchBucketEncryptionConfiguration
}
return config.DeclaredEncryption, s3err.ErrNone
}
// updateEncryptionConfiguration updates the encryption configuration for a bucket
func (s3a *S3ApiServer) updateEncryptionConfiguration(bucket string, encryptionConfig *s3_pb.EncryptionConfiguration) s3err.ErrorCode {
// Update using structured API
@@ -249,6 +265,10 @@ func (s3a *S3ApiServer) removeEncryptionConfiguration(bucket string) s3err.Error
}
if metadata.Encryption == nil {
// A bucket's declared encryption is fixed when the bucket is created.
if declared, _ := s3a.declaredEncryption(bucket); declared != nil {
return s3err.ErrMethodNotAllowed
}
return s3err.ErrNoSuchBucketEncryptionConfiguration
}
+15
View File
@@ -359,6 +359,7 @@ const (
contextKeyOriginalRequestPath contextKey = "s3-original-request-path"
contextKeyAccessKeyId contextKey = "s3-access-key-id"
contextKeyStoredAccessKeyId contextKey = "s3-stored-access-key-id"
contextKeyRequestBodyLimit contextKey = "s3-request-body-limit"
)
// identityHolder is a mutable container for the authenticated identity name,
@@ -529,6 +530,20 @@ func GetOriginalRequestPathFromContext(r *http.Request) string {
return ""
}
// SetRequestBodyLimitInContext sets how much of the body SigV4 verification
// reads. The route sets it: the credential scope is not verified yet.
func SetRequestBodyLimitInContext(ctx context.Context, limit int64) context.Context {
return context.WithValue(ctx, contextKeyRequestBodyLimit, limit)
}
// GetRequestBodyLimitFromContext returns the route's limit, or def.
func GetRequestBodyLimitFromContext(r *http.Request, def int64) int64 {
if limit, ok := r.Context().Value(contextKeyRequestBodyLimit).(int64); ok {
return limit
}
return def
}
// SetAccessKeyIdInContext stores the access key id the request authenticated
// with: a static key's id, or an STS session's temporary key id. Several keys
// can belong to one identity, so the identity name alone cannot tell their
+3
View File
@@ -13,6 +13,9 @@ var (
Separator = ":"
)
// S3VectorsMaxRequestBodyBytes caps the body of an S3 Vectors API request.
const S3VectorsMaxRequestBodyBytes = 32 << 20
func Concat(elements ...string) string {
return strings.Join(elements, Separator)
}
+7
View File
@@ -778,6 +778,13 @@ func (km *SSES3KeyManager) StoreKey(key *SSES3Key) {
// The DEK is encrypted with the super key and stored in object metadata
}
// IsConfigured reports whether a KEK is loaded, i.e. SSE-S3 can encrypt.
func (km *SSES3KeyManager) IsConfigured() bool {
km.mu.RLock()
defer km.mu.RUnlock()
return len(km.superKey) > 0
}
// GetMasterKey returns a derived key from the master KEK for STS signing
// This uses HKDF to isolate the STS security domain from the SSE-S3 domain
func (km *SSES3KeyManager) GetMasterKey() []byte {
+5
View File
@@ -24,6 +24,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
// BucketConfig represents cached bucket configuration. Only fields parsed
@@ -56,6 +57,7 @@ type BucketConfig struct {
LifecycleTransitionMinSize string
Tags map[string]string // parsed from entry content
Encryption *s3_pb.EncryptionConfiguration // parsed from entry content
DeclaredEncryption *s3_pb.EncryptionConfiguration // declared by a table bucket's record
KMSKeyCache *BucketKMSCache // Per-bucket KMS key cache for SSE-KMS operations
LastModified time.Time
}
@@ -441,6 +443,9 @@ func (s3a *S3ApiServer) newBucketConfigFromEntry(bucket string, entry *filer_pb.
config.ObjectLockConfig = objectLockConfig
}
config.BucketPolicy = loadBucketPolicyFromExtended(entry, bucket)
if algorithm := s3tables.DeclaredBucketEncryption(entry); algorithm != "" {
config.DeclaredEncryption = &s3_pb.EncryptionConfiguration{SseAlgorithm: algorithm}
}
if lifecycleXML, exists := entry.Extended[bucketLifecycleConfigurationXMLKey]; exists && len(lifecycleXML) > 0 {
config.LifecycleXML = lifecycleXML
@@ -2,6 +2,7 @@ package s3api
import (
"bytes"
"errors"
"net/http"
"net/http/httptest"
"strings"
@@ -12,6 +13,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
// Issue 11647: Bucket default encryption (SSE-S3) must be applied when
@@ -218,3 +220,72 @@ func TestPutObjectUnderUploadsFolderKeepsBucketDefaultEncryption(t *testing.T) {
}
}
}
// A vector bucket is a LANCE table bucket; the AES256 its record declares is
// the bucket's default encryption. An ICEBERG bucket's declaration is not
// applied, because its catalog reads client manifests straight from the filer.
func TestPutObjectAppliesTableBucketDeclaredEncryption(t *testing.T) {
km := GetSSES3KeyManager()
oldSuperKey := km.superKey
km.superKey = make([]byte, 32)
for i := range km.superKey {
km.superKey[i] = byte(i + 1)
}
t.Cleanup(func() {
km.superKey = oldSuperKey
})
volume := startFakeVolumeServer(t)
filerImpl := &ambiguousPutFiler{
volume: volume,
entries: map[string]*filer_pb.Entry{},
apply: true,
}
s3a := newPutTestServer(t, startFakeFiler(t, filerImpl))
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
for bucket, format := range map[string]string{"lance": s3tables.FormatLance, "ice": s3tables.FormatIceberg} {
s3a.bucketConfigCache.Set(bucket, s3a.newBucketConfigFromEntry(bucket, &filer_pb.Entry{
Name: bucket,
Extended: map[string][]byte{
s3tables.ExtendedKeyTableBucket: []byte("true"),
s3tables.ExtendedKeyMetadata: []byte(`{"name":"` + bucket + `","format":"` + format +
`","encryptionConfiguration":{"sseAlgorithm":"AES256"}}`),
},
}))
}
cfg, err := s3a.GetBucketEncryptionConfig("lance")
if err != nil || cfg.SseAlgorithm != EncryptionTypeAES256 {
t.Fatalf("GetBucketEncryptionConfig(lance) = %v, %v; want AES256", cfg, err)
}
if metadata, err := s3a.GetBucketMetadata("lance"); err != nil || metadata.Encryption != nil {
t.Fatalf("declared encryption leaked into the stored bucket metadata: %v, %v", metadata, err)
}
if _, err := s3a.GetBucketEncryptionConfig("ice"); !errors.Is(err, ErrNoEncryptionConfig) {
t.Fatalf("GetBucketEncryptionConfig(ice) error = %v, want ErrNoEncryptionConfig", err)
}
if code := s3a.removeEncryptionConfiguration("lance"); code != s3err.ErrMethodNotAllowed {
t.Fatalf("DeleteBucketEncryption(lance) = %v, want MethodNotAllowed for a declared encryption", code)
}
if code := s3a.removeEncryptionConfiguration("ice"); code != s3err.ErrNoSuchBucketEncryptionConfiguration {
t.Fatalf("DeleteBucketEncryption(ice) = %v, want NoSuchBucketEncryptionConfiguration", code)
}
for bucket, want := range map[string]filer_pb.SSEType{"lance": filer_pb.SSEType_SSE_S3, "ice": filer_pb.SSEType_NONE} {
filePath := "/buckets/" + bucket + "/vector/idx/data/0.lance"
r := httptest.NewRequest(http.MethodPut, "/"+bucket+"/vector/idx/data/0.lance", nil)
_, code, _ := s3a.putToFiler(r, filePath, strings.NewReader("lance data file"), bucket, "vector/idx/data/0.lance", 1, 0, nil, false, "")
if code != s3err.ErrNone {
t.Fatalf("putToFiler(%s) = %v, want %v", bucket, code, s3err.ErrNone)
}
entry := filerImpl.entries[filePath]
if entry == nil || len(entry.Chunks) == 0 {
t.Fatalf("no entry with chunks at %s", filePath)
}
for i, chunk := range entry.Chunks {
if chunk.SseType != want {
t.Errorf("%s chunk %d: SseType %v, want %v", bucket, i, chunk.SseType, want)
}
}
}
}
+2
View File
@@ -342,6 +342,8 @@ func iamAuthErrorCode(errCode s3err.ErrorCode) string {
return "RequestTimeTooSkewed"
case s3err.ErrInternalError:
return iam.ErrCodeServiceFailureException
case s3err.ErrEntityTooLarge:
return "InvalidInput"
default:
return "IncompleteSignature"
}
+1
View File
@@ -3041,5 +3041,6 @@ func TestIamAuthErrorCode(t *testing.T) {
assert.Equal(t, "SignatureDoesNotMatch", iamAuthErrorCode(s3err.ErrSignatureDoesNotMatch))
assert.Equal(t, "AccessDenied", iamAuthErrorCode(s3err.ErrAccessDenied))
assert.Equal(t, "RequestTimeTooSkewed", iamAuthErrorCode(s3err.ErrRequestTimeTooSkewed))
assert.Equal(t, "InvalidInput", iamAuthErrorCode(s3err.ErrEntityTooLarge))
assert.Equal(t, "IncompleteSignature", iamAuthErrorCode(s3err.ErrMissingFields))
}
+17 -2
View File
@@ -1106,9 +1106,9 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
// GetBucketLifecycleConfiguration
bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLifecycleConfigurationHandler, ACTION_READ)), "GET")).Queries("lifecycle", "")
// PutBucketLifecycleConfiguration
bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketLifecycleConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("lifecycle", "")
bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketLifecycleConfigurationHandler, ACTION_ADMIN)), "PUT")).Queries("lifecycle", "")
// DeleteBucketLifecycleConfiguration
bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketLifecycleHandler, ACTION_WRITE)), "DELETE")).Queries("lifecycle", "")
bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketLifecycleHandler, ACTION_ADMIN)), "DELETE")).Queries("lifecycle", "")
// GetBucketLocation
bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLocationHandler, ACTION_READ)), "GET")).Queries("location", "")
@@ -1500,6 +1500,21 @@ func (s3a *S3ApiServer) AuthenticateRequest(r *http.Request) (string, interface{
return "", nil, err
}
// FilerSigningKey is the current jwt.filer_signing.key, which follows a
// security.toml reload.
func (s3a *S3ApiServer) FilerSigningKey() security.SigningKey {
if s3a.filerGuard == nil {
return nil
}
return s3a.filerGuard.SigningKey()
}
// AuthEnabled reports whether requests must carry valid credentials. With no
// identities configured, every request is allowed as the admin.
func (s3a *S3ApiServer) AuthEnabled() bool {
return s3a.iam == nil || s3a.iam.isEnabled()
}
// DefaultAllow returns whether access is allowed by default when no policy is found
func (s3a *S3ApiServer) DefaultAllow() bool {
if s3a.iam == nil || s3a.iam.iamIntegration == nil {
+59
View File
@@ -5,6 +5,8 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -529,3 +531,60 @@ func TestRouting_BucketRootPreflightUsesBucketCORS(t *testing.T) {
})
}
}
// Write covers a bucket's objects, not its configuration: an identity holding
// Write on the bucket is refused lifecycle writes, while Admin on the bucket or
// a policy granting s3:PutLifecycleConfiguration passes. The bucket is
// negatively cached, so a request that gets past authorization ends at the
// handler's NoSuchBucket.
func TestRouting_BucketLifecycleWritesNeedAdmin(t *testing.T) {
const bucket = "config-bucket"
configFile := filepath.Join(t.TempDir(), "s3.json")
require.NoError(t, os.WriteFile(configFile, []byte(`{
"identities": [
{"name":"writer","credentials":[{"accessKey":"WRITERKEY","secretKey":"writersecret"}],"actions":["Read:`+bucket+`","Write:`+bucket+`","List:`+bucket+`","Tagging:`+bucket+`"]},
{"name":"bucketadmin","credentials":[{"accessKey":"BUCKETADMINKEY","secretKey":"bucketadminsecret"}],"actions":["Admin:`+bucket+`"]},
{"name":"lifecycler","credentials":[{"accessKey":"LIFECYCLERKEY","secretKey":"lifecyclersecret"}],"policyNames":["LifecyclePolicy"]}
],
"policies": [
{"name":"LifecyclePolicy","content":"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:PutLifecycleConfiguration\"],\"Resource\":[\"arn:aws:s3:::`+bucket+`\"]}]}"}
]
}`), 0600))
s3a := setupRoutingTestServer(t)
s3a.iam = NewIdentityAccessManagementWithStore(&S3ApiServerOption{Config: configFile}, nil, "memory")
require.True(t, s3a.iam.isEnabled())
s3a.cb = &CircuitBreaker{s3a: s3a}
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
s3a.bucketConfigCache.SetNegativeCache(bucket)
router := mux.NewRouter()
s3a.registerRouter(router)
send := func(accessKey, secretKey, method, body string) *httptest.ResponseRecorder {
req := httptest.NewRequest(method, "http://localhost:8333/"+bucket+"?lifecycle", strings.NewReader(body))
creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
_, err := v4.NewSigner(creds).Sign(req, strings.NewReader(body), "s3", "us-east-1", time.Now())
require.NoError(t, err)
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
return rr
}
const lifecycleXML = `<LifecycleConfiguration><Rule><ID>r</ID><Status>Enabled</Status><Filter><Prefix>p/</Prefix></Filter><Expiration><Days>1</Days></Expiration></Rule></LifecycleConfiguration>`
for _, tc := range []struct {
method, body string
}{
{http.MethodPut, lifecycleXML},
{http.MethodDelete, ""},
} {
rr := send("WRITERKEY", "writersecret", tc.method, tc.body)
assert.Equal(t, http.StatusForbidden, rr.Code, "writer %s", tc.method)
assert.Contains(t, rr.Body.String(), "AccessDenied", "writer %s", tc.method)
for _, key := range [][2]string{{"BUCKETADMINKEY", "bucketadminsecret"}, {"LIFECYCLERKEY", "lifecyclersecret"}} {
rr = send(key[0], key[1], tc.method, tc.body)
assert.Equal(t, http.StatusNotFound, rr.Code, "%s %s", key[0], tc.method)
assert.Contains(t, rr.Body.String(), "NoSuchBucket", "%s %s", key[0], tc.method)
}
}
}
+6
View File
@@ -58,6 +58,11 @@ func (st *S3TablesApiServer) SetLocalAccountChecker(fn func(string) bool) {
st.handler.SetLocalAccountChecker(fn)
}
// SetEncryptionChecker sets the handler's SSE-S3 availability predicate.
func (st *S3TablesApiServer) SetEncryptionChecker(fn func() bool) {
st.handler.SetEncryptionChecker(fn)
}
// S3TablesHandler handles S3 Tables API requests
func (st *S3TablesApiServer) S3TablesHandler(w http.ResponseWriter, r *http.Request) {
st.handler.HandleRequest(w, r, st)
@@ -72,6 +77,7 @@ func (st *S3TablesApiServer) WithFilerClient(streamingMode bool, fn func(filer_p
func (s3a *S3ApiServer) registerS3TablesRoutes(router *mux.Router) {
// Create S3 Tables handler
s3TablesApi := NewS3TablesApiServer(s3a)
s3TablesApi.SetEncryptionChecker(GetSSES3KeyManager().IsConfigured)
if s3a.iam != nil {
s3TablesApi.SetLocalAccountChecker(s3a.iam.isConfiguredAccountID)
}
+28
View File
@@ -159,6 +159,34 @@ func TestLanceAuthRejectsFailedSignature(t *testing.T) {
assert.NotEqual(t, http.StatusOK, rr.Code)
}
// The Lance catalog takes any x-api-key while the S3 port runs without
// identities, as the admin, and refuses an unknown key once auth is on.
func TestLanceAuthTakesAnyKeyOnlyWhileS3AuthIsOff(t *testing.T) {
s3a := &S3ApiServer{iam: &IdentityAccessManagement{}}
server := lance.NewServer(nil, s3a)
server.SetCredentialValidator(s3a)
call := func() (int, string) {
req := httptest.NewRequest(http.MethodGet, "/v1/namespace/list", nil)
req.Header.Set("x-api-key", "unknown:key")
identity := ""
rr := httptest.NewRecorder()
server.Auth(func(w http.ResponseWriter, r *http.Request) {
identity = s3_constants.GetIdentityNameFromContext(r)
w.WriteHeader(http.StatusOK)
})(rr, req)
return rr.Code, identity
}
code, identity := call()
assert.Equal(t, http.StatusOK, code)
assert.Equal(t, s3_constants.AccountAdminId, identity)
s3a.iam.isAuthEnabled = true
code, _ = call()
assert.Equal(t, http.StatusUnauthorized, code)
}
func TestAuthSignatureOnlyScrubsAccountHeader(t *testing.T) {
s3a := setupDefaultAllowAuthServer(t)
+326 -21
View File
@@ -11,8 +11,10 @@ import (
"path"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
@@ -107,6 +109,20 @@ func (h *S3TablesHandler) mutateEntry(
client filer_pb.SeaweedFilerClient,
path string,
mutate func(entry *filer_pb.Entry) error,
) error {
return h.mutateEntryAsserting(ctx, client, path, nil, mutate)
}
// mutateEntryAsserting is mutateEntry with the keys in absent asserted at the
// write too, as this package's own attributes are: one absent when the entry was
// read and written by someone else since fails the compare, instead of being
// written over with the entry as it was read.
func (h *S3TablesHandler) mutateEntryAsserting(
ctx context.Context,
client filer_pb.SeaweedFilerClient,
path string,
absent []string,
mutate func(entry *filer_pb.Entry) error,
) error {
dir, name := splitPath(path)
@@ -120,7 +136,7 @@ func (h *S3TablesHandler) mutateEntry(
}
entry := resp.Entry
expected := snapshotExtended(entry.Extended)
expected := snapshotExtended(entry.Extended, absent...)
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
@@ -401,13 +417,25 @@ func (h *S3TablesHandler) getExtendedAttribute(ctx context.Context, client filer
// readNamespaceMetadataAttr returns the namespace-metadata xattr at
// namespacePath. A directory without it is not a namespace — a delete that kept
// the data under it leaves one — so its absence reads as a missing entry.
// the data under it leaves one — so its absence reads as a missing entry. So
// does a namespace still marked by its create: like a table's, it is absent to
// every reader until the create confirms it.
func (h *S3TablesHandler) readNamespaceMetadataAttr(ctx context.Context, client filer_pb.SeaweedFilerClient, namespacePath string) ([]byte, error) {
data, err := h.getExtendedAttribute(ctx, client, namespacePath, ExtendedKeyMetadata)
if errors.Is(err, ErrAttributeNotFound) {
entry, err := h.lookupEntry(ctx, client, namespacePath)
if err != nil {
return nil, err
}
data, ok := entry.Extended[ExtendedKeyMetadata]
if !ok || isUnfinishedCreate(entry.Extended) {
return nil, fmt.Errorf("%w: %s", filer_pb.ErrNotFound, namespacePath)
}
return data, err
return data, nil
}
// isFinishedNamespaceEntry is isNamespaceEntry for a namespace its create has
// confirmed, the only kind a reader or a delete may act on.
func isFinishedNamespaceEntry(entry *filer_pb.Entry) bool {
return isNamespaceEntry(entry) && !isUnfinishedCreate(entry.Extended)
}
// loadNamespaceMetadata resolves a namespace to its metadata. A directory that
@@ -455,14 +483,6 @@ func (h *S3TablesHandler) lookupEntry(ctx context.Context, client filer_pb.Seawe
return resp.Entry, nil
}
// deleteExtendedAttribute deletes an extended attribute from an entry
func (h *S3TablesHandler) deleteExtendedAttribute(ctx context.Context, client filer_pb.SeaweedFilerClient, path, key string) error {
return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
delete(extended, key)
return nil
})
}
// deleteDirectory deletes a directory and all its contents
func (h *S3TablesHandler) deleteDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error {
dir, name := splitPath(path)
@@ -499,9 +519,46 @@ func (h *S3TablesHandler) deleteDirectoryIfMatches(ctx context.Context, client f
return nil
}
// errDirNotEmpty is a directory that holds something the caller meant to leave
// alone: a file in a leftover tree, or an entry the filer found in a directory
// it was asked to delete without its contents.
var errDirNotEmpty = errors.New("directory is not empty")
// deleteEmptyDirectoryIfMatches deletes a directory without its contents, only
// while it still carries the extended attributes the caller captured. The
// filer checks both under the path lock, in the one delete: a changed
// attribute is ErrConcurrentUpdate, and an entry that appeared in the
// directory is errDirNotEmpty, so neither a directory taken back since the
// caller looked at it nor a file written into it since is lost. The mark of an
// empty S3 object stored on the directory is asserted absent too, so one
// written since is kept the same way.
func (h *S3TablesHandler) deleteEmptyDirectoryIfMatches(ctx context.Context, client filer_pb.SeaweedFilerClient, path string, expected *filer_pb.Entry) error {
dir, name := splitPath(path)
resp, err := client.DeleteEntry(ctx, &filer_pb.DeleteEntryRequest{
Directory: dir,
Name: name,
IsDeleteData: true,
ExpectedExtended: snapshotExtended(expected.Extended, s3_constants.SeaweedFSPrefixObject),
})
if status.Code(err) == codes.FailedPrecondition {
return fmt.Errorf("%w: %s", ErrConcurrentUpdate, path)
}
if err != nil {
return err
}
if resp != nil && resp.Error != "" {
if err := filer.DeleteEntryError(resp.Error); !errors.Is(err, filer.ErrNonEmptyFolder) {
return err
}
return fmt.Errorf("%w: %s", errDirNotEmpty, path)
}
return nil
}
var (
errParentDeleting = errors.New("parent is being deleted")
errParentGone = errors.New("parent was deleted")
errParentCreating = errors.New("parent is still being created")
errNoDeleteClaim = errors.New("claim not held")
errBackOutFailed = errors.New("backing out the create failed")
errSourceGone = errors.New("rename source was deleted")
@@ -527,18 +584,71 @@ func (h *S3TablesHandler) claimForDelete(ctx context.Context, client filer_pb.Se
return claim, claimed, nil
}
// releaseDeleteClaim removes claim from path while path still holds it, even
// after the request was cancelled.
func (h *S3TablesHandler) releaseDeleteClaim(ctx context.Context, client filer_pb.SeaweedFilerClient, path, claim string) {
err := h.mutateEntryExtended(context.WithoutCancel(ctx), client, path, func(extended map[string][]byte) error {
// tryReleaseDeleteClaim removes claim from path once, while path still holds
// it. A claim another delete replaced or lifted, or an entry that is gone,
// leaves nothing to release and is not an error.
func (h *S3TablesHandler) tryReleaseDeleteClaim(ctx context.Context, client filer_pb.SeaweedFilerClient, path, claim string) error {
err := h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
if string(extended[ExtendedKeyDeleting]) != claim {
return errNoDeleteClaim
}
delete(extended, ExtendedKeyDeleting)
return nil
})
if err != nil && !errors.Is(err, errNoDeleteClaim) && !errors.Is(err, filer_pb.ErrNotFound) {
glog.Warningf("release delete claim on %s: %v", path, err)
if errors.Is(err, errNoDeleteClaim) || errors.Is(err, filer_pb.ErrNotFound) {
return nil
}
return err
}
// releaseDeleteClaim removes claim from path while path still holds it, even
// after the request was cancelled. A claim left behind refuses creates under a
// bucket or namespace and makes a leftover directory read as taken, so a failed
// release is retried, including one that lost to another write of the entry.
func (h *S3TablesHandler) releaseDeleteClaim(ctx context.Context, client filer_pb.SeaweedFilerClient, path, claim string) {
ctx = context.WithoutCancel(ctx)
var err error
for attempt := 0; attempt < extendedAttributeUpdateAttempts; attempt++ {
if attempt > 0 {
time.Sleep(100 * time.Millisecond)
}
if err = h.tryReleaseDeleteClaim(ctx, client, path, claim); err == nil {
return
}
}
glog.Warningf("release delete claim on %s: %v", path, err)
}
// isTableBucketCreatedAt accepts only the table bucket created at createdAt. A
// bucket deleted and created again under the same name has a CreatedAt of its
// own, so a create authorized against the first reads the second as gone.
func isTableBucketCreatedAt(createdAt time.Time) func(*filer_pb.Entry) bool {
return func(entry *filer_pb.Entry) bool {
if !IsTableBucketEntry(entry) {
return false
}
var metadata tableBucketMetadata
if json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata) != nil {
return false
}
return metadata.CreatedAt.Equal(createdAt)
}
}
// isNamespaceCreatedAt is isTableBucketCreatedAt for a namespace its create has
// confirmed: one deleted and created again under the same name reads as the
// authorized one gone. A bucket created again holds only namespaces of its own,
// so a create bound to its namespace this way is bound to the bucket too.
func isNamespaceCreatedAt(createdAt time.Time) func(*filer_pb.Entry) bool {
return func(entry *filer_pb.Entry) bool {
if !isFinishedNamespaceEntry(entry) {
return false
}
var metadata namespaceMetadata
if json.Unmarshal(entry.Extended[ExtendedKeyMetadata], &metadata) != nil {
return false
}
return metadata.CreatedAt.Equal(createdAt)
}
}
@@ -549,7 +659,8 @@ func isNamespaceEntry(entry *filer_pb.Entry) bool {
}
// checkParentOpen returns errParentGone if parentPath is missing or not a
// catalog entry, and errParentDeleting if a delete has claimed it.
// catalog entry, errParentCreating if the create that wrote it has not
// confirmed it yet, and errParentDeleting if a delete has claimed it.
func (h *S3TablesHandler) checkParentOpen(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool) error {
_, err := h.openParent(ctx, client, parentPath, isParent)
return err
@@ -568,6 +679,9 @@ func (h *S3TablesHandler) openParent(ctx context.Context, client filer_pb.Seawee
if !isParent(entry) {
return nil, errParentGone
}
if isUnfinishedCreate(entry.Extended) {
return nil, errParentCreating
}
if _, claimed := entry.Extended[ExtendedKeyDeleting]; claimed {
return nil, errParentDeleting
}
@@ -636,6 +750,28 @@ func (h *S3TablesHandler) mutateCatalogRecord(
})
}
// requireTableCreatedAt refuses a write to the record at path unless it is the
// table created at createdAt, the one the caller was authorized against: a
// table deleted and created again under the name has a CreatedAt of its own.
// Only the creation time is compared, so the commits an Iceberg writer lands
// meanwhile do not turn a write of the table's policy, tags or maintenance
// settings into a conflict.
func requireTableCreatedAt(extended map[string][]byte, path string, createdAt time.Time) error {
var current tableMetadataInternal
if json.Unmarshal(extended[ExtendedKeyMetadata], &current) != nil || !current.CreatedAt.Equal(createdAt) {
return fmt.Errorf("%w: %s holds another table", ErrConcurrentUpdate, path)
}
return nil
}
// requireTableBucketCreatedAt is requireTableCreatedAt for a table bucket.
func requireTableBucketCreatedAt(extended map[string][]byte, path string, createdAt time.Time) error {
if !isTableBucketCreatedAt(createdAt)(&filer_pb.Entry{Extended: extended}) {
return fmt.Errorf("%w: %s holds another table bucket", ErrConcurrentUpdate, path)
}
return nil
}
// confirmParent re-checks the parent after a create wrote written onto
// childPath. If the parent is open, it clears the record's unfinished mark —
// then checks the parent once more, since a delete claiming it in between
@@ -644,6 +780,20 @@ func (h *S3TablesHandler) mutateCatalogRecord(
// it removes what the create wrote and returns why, or errBackOutFailed if
// the record stays.
func (h *S3TablesHandler) confirmParent(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool, childPath string, written map[string][]byte, emptyDirs ...string) error {
return h.confirmChild(ctx, client, parentPath, isParent, true, childPath, written, emptyDirs)
}
// confirmParentOnce is confirmParent for a record that other creates build on,
// a namespace. Checking the parent again after the mark comes off would leave a
// window in which a child is created in the record and the check then removes
// the record under it, so the parent is checked once, before the mark comes
// off. That is enough while a delete of the parent counts the marked record: one
// claiming the parent after the check scans the record and is refused.
func (h *S3TablesHandler) confirmParentOnce(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool, childPath string, written map[string][]byte, emptyDirs ...string) error {
return h.confirmChild(ctx, client, parentPath, isParent, false, childPath, written, emptyDirs)
}
func (h *S3TablesHandler) confirmChild(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath string, isParent func(*filer_pb.Entry) bool, recheck bool, childPath string, written map[string][]byte, emptyDirs []string) error {
parent, err := h.openParent(ctx, client, parentPath, isParent)
confirmed := false
if err == nil {
@@ -651,7 +801,9 @@ func (h *S3TablesHandler) confirmParent(ctx context.Context, client filer_pb.Sea
// create wrote; a failed write backs the record out like a closed parent.
if err = h.removeExtendedAttributesIf(ctx, client, childPath, written, ExtendedKeyDeleting); err == nil {
confirmed = true
err = h.parentStillOpen(ctx, client, parentPath, isParent, parent)
if recheck {
err = h.parentStillOpen(ctx, client, parentPath, isParent, parent)
}
}
}
if err == nil {
@@ -753,6 +905,159 @@ func (h *S3TablesHandler) dirHoldsFiles(ctx context.Context, client filer_pb.Sea
return false, nil
}
// claimTreeDirectory writes a fresh delete claim onto the directory at dirPath
// while it still carries expected, and returns the claim with the attributes it
// left. A claim seen in expected, one a delete that died left, is replaced like
// any other attribute; one written since fails the compare.
//
// The claim is what ties the later delete to this directory: the delete
// requires it, and only this directory carries it. A file or an S3 object at
// dirPath is refused with errDirNotEmpty, whether it was there when the entry
// was read or took the directory's place before the write, which the filer
// refuses since it never writes a directory entry over a file. One that takes
// the place after the write lacks the claim, so the delete is refused instead.
// An S3 object landing on the directory between the read and the write carries
// the prefix-object mark, which the write asserts absent, so the write fails
// instead of putting the bare directory back over the object.
func (h *S3TablesHandler) claimTreeDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, dirPath string, expected map[string][]byte) (string, map[string][]byte, error) {
claim := generateVersionToken()
var claimed map[string][]byte
err := h.mutateEntryAsserting(ctx, client, dirPath, []string{s3_constants.SeaweedFSPrefixObject}, func(entry *filer_pb.Entry) error {
if !entry.IsDirectory || entry.IsDirectoryKeyObject() {
return fmt.Errorf("%w: %s is an object", errDirNotEmpty, dirPath)
}
for key, want := range expected {
if !bytes.Equal(entry.Extended[key], want) {
return fmt.Errorf("%w: %s changed on %s", ErrConcurrentUpdate, key, dirPath)
}
}
entry.Extended[ExtendedKeyDeleting] = []byte(claim)
claimed = snapshotExtended(entry.Extended)
return nil
})
if errors.Is(err, filer_pb.ErrNotFound) {
return "", nil, fmt.Errorf("%w: %s", ErrConcurrentUpdate, dirPath)
}
if err != nil && !errors.Is(err, errDirNotEmpty) && !errors.Is(err, ErrConcurrentUpdate) {
// The filer reports the refused write over a file only as an opaque
// failure, so look at what holds the path now.
if current, lookupErr := h.lookupEntry(ctx, client, dirPath); lookupErr == nil && !current.IsDirectory {
return "", nil, fmt.Errorf("%w: %s is an object", errDirNotEmpty, dirPath)
}
}
if err != nil {
return "", nil, err
}
return claim, claimed, nil
}
// removeTreeDirectory removes a directory the walk listed as listed, under a
// claim of its own, and lifts the claim if the delete is refused.
func (h *S3TablesHandler) removeTreeDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, dirPath string, listed *filer_pb.Entry) error {
claim, claimed, err := h.claimTreeDirectory(ctx, client, dirPath, snapshotExtended(listed.Extended))
if err != nil {
return err
}
if err := h.deleteEmptyDirectoryIfMatches(ctx, client, dirPath, &filer_pb.Entry{Extended: claimed}); err != nil {
h.releaseDeleteClaim(ctx, client, dirPath, claim)
return err
}
return nil
}
// removeEmptyTree removes dirPath and every directory under it, provided no
// file, no S3 object stored on a directory entry, and no catalog record sits
// anywhere in the tree, dirPath included. dirEntry is dirPath as the caller saw
// it.
//
// dirPath is claimed first, in a write that requires the attributes the caller
// saw: a directory taken back as a namespace since then fails it, before
// anything under it is touched, and the claim keeps it from being taken back
// while the tree goes. With dirPath no namespace, nothing under it can become a
// catalog record either, so a record the walk meets was there before and is
// refused, not removed.
//
// The walk lists the tree a page at a time and stops at the first file, object
// or record with errDirNotEmpty, before anything is removed. Directories then go
// deepest first, each claimed with the attributes it had when listed and
// deleted under that claim, which refuses a directory that holds an entry by
// now or was replaced by a file. dirPath goes last, still under its claim; any
// other way out lifts the claim.
func (h *S3TablesHandler) removeEmptyTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dirPath string, dirEntry *filer_pb.Entry) (err error) {
claim, claimed, err := h.claimTreeDirectory(ctx, client, dirPath, snapshotExtended(dirEntry.Extended))
if err != nil {
return err
}
defer func() {
if err != nil {
h.releaseDeleteClaim(ctx, client, dirPath, claim)
}
}()
type listedDir struct {
path string
entry *filer_pb.Entry
}
listed := []listedDir{{dirPath, &filer_pb.Entry{Extended: claimed}}}
walkCtx, cancel := context.WithCancel(ctx)
defer cancel()
// A directory is listed after its parent, so removing in reverse order
// removes every directory after the ones under it.
for next := 0; next < len(listed); next++ {
dir := listed[next].path
lastFileName := ""
for {
resp, err := client.ListEntries(walkCtx, &filer_pb.ListEntriesRequest{
Directory: dir,
Limit: dirListPageSize,
StartFromFileName: lastFileName,
InclusiveStartFrom: lastFileName == "",
})
if err != nil {
if status.Code(err) == codes.NotFound {
break // gone, so its delete below reports the change
}
return err
}
received := false
for {
entry, err := resp.Recv()
if err != nil {
if err == io.EOF || status.Code(err) == codes.NotFound {
break
}
return err
}
if entry.Entry == nil {
continue
}
received = true
lastFileName = entry.Entry.Name
child := path.Join(dir, entry.Entry.Name)
if !entry.Entry.IsDirectory || entry.Entry.IsDirectoryKeyObject() {
return fmt.Errorf("%w: %s is an object", errDirNotEmpty, child)
}
if _, isRecord := entry.Entry.Extended[ExtendedKeyMetadata]; isRecord {
return fmt.Errorf("%w: %s is a catalog record", errDirNotEmpty, child)
}
listed = append(listed, listedDir{child, entry.Entry})
}
if !received {
break
}
}
}
for i := len(listed) - 1; i > 0; i-- {
if err := h.removeTreeDirectory(ctx, client, listed[i].path, listed[i].entry); err != nil {
return err
}
}
return h.deleteEmptyDirectoryIfMatches(ctx, client, dirPath, &filer_pb.Entry{Extended: claimed})
}
// keepAsLeftover clears a namespace's catalog attributes and the delete's
// claim in one write, while the entry still holds claimed, keeping the
// directory and the data under it. The write compares against claimed so the
Loaded 100 of 155 files, more files were not shown because too many files have changed in this diff. Show more