diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index 45619a21e0..4336a3ce49 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -8,6 +8,7 @@ on: - 'seaweed-volume/**' - 'seaweed-common/**' - 'seaweed-worker/**' + - 'seaweed-vector/**' - 'docker/**' - 'go.mod' - 'go.sum' @@ -50,7 +51,8 @@ jobs: ~/.cargo/git seaweed-volume/target seaweed-worker/target/${{ matrix.target }}/release - key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} + seaweed-vector/target/${{ matrix.target }}/release + key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }} restore-keys: | rust-docker-dev-${{ matrix.target }}- @@ -83,6 +85,12 @@ jobs: cargo build --release -p weed-lance-worker --target ${{ matrix.target }} cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Build the vector engine + run: | + cd seaweed-vector + cargo build --release -p weed-vector-server --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: @@ -90,6 +98,7 @@ jobs: path: | weed-volume-normal-${{ matrix.arch }} weed-worker-${{ matrix.arch }} + weed-vector-${{ matrix.arch }} build-dev-containers: needs: [build-rust-binaries] @@ -127,8 +136,17 @@ jobs: echo "Placed pre-built Rust worker for ${arch}" fi done + mkdir -p docker/weed-vector-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-vector-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}" + echo "Placed pre-built vector engine for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ ls -la docker/weed-worker-prebuilt/ + ls -la docker/weed-vector-prebuilt/ - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 668c6046f6..f5a445e07a 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -96,7 +96,8 @@ jobs: ~/.cargo/git seaweed-volume/target seaweed-worker/target/${{ matrix.target }}/release - key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} + seaweed-vector/target/${{ matrix.target }}/release + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }} restore-keys: | rust-docker-${{ matrix.target }}- @@ -137,6 +138,17 @@ jobs: cargo build --release -p weed-lance-worker --target ${{ matrix.target }} cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Build the vector engine + run: | + # An older source_ref predates the engine; its image goes without one. + if [ ! -f seaweed-vector/crates/server/Cargo.toml ]; then + echo "No vector engine in this source ref" + exit 0 + fi + cd seaweed-vector + cargo build --release -p weed-vector-server --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: @@ -145,6 +157,7 @@ jobs: weed-volume-large-disk-${{ matrix.arch }} weed-volume-normal-${{ matrix.arch }} weed-worker-${{ matrix.arch }} + weed-vector-${{ matrix.arch }} build: needs: [setup, build-rust-binaries] @@ -217,8 +230,17 @@ jobs: echo "Placed pre-built Rust worker for ${arch}" fi done + mkdir -p docker/weed-vector-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-vector-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}" + echo "Placed pre-built vector engine for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ ls -la docker/weed-worker-prebuilt/ + ls -la docker/weed-vector-prebuilt/ - name: Docker meta id: docker_meta @@ -342,8 +364,17 @@ jobs: echo "Placed pre-built Rust worker for ${arch}" fi done + mkdir -p docker/weed-vector-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-vector-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}" + echo "Placed pre-built vector engine for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ ls -la docker/weed-worker-prebuilt/ + ls -la docker/weed-vector-prebuilt/ - name: Create BuildKit config for local scan build if: needs.setup.outputs.publish != 'true' run: | diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index c78e5da7dc..47f040fd83 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -43,9 +43,9 @@ concurrency: jobs: # ── Pre-build the Rust binaries natively ──────────────────────────── - # The volume server and the Rust maintenance worker, cross-compiled for - # amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into - # ~15 minutes of native compilation. + # The volume server, the Rust maintenance worker and the vector engine, + # cross-compiled for amd64 and arm64 without QEMU, turning a 5-hour emulated + # cargo build into ~15 minutes of native compilation. build-rust-binaries: runs-on: ubuntu-24.04 strategy: @@ -73,7 +73,8 @@ jobs: ~/.cargo/git seaweed-volume/target seaweed-worker/target/${{ matrix.target }}/release - key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} + seaweed-vector/target/${{ matrix.target }}/release + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }} restore-keys: | rust-docker-${{ matrix.target }}- @@ -114,6 +115,12 @@ jobs: cargo build --release -p weed-lance-worker --target ${{ matrix.target }} cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Build the vector engine + run: | + cd seaweed-vector + cargo build --release -p weed-vector-server --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: @@ -122,6 +129,7 @@ jobs: weed-volume-large-disk-${{ matrix.arch }} weed-volume-normal-${{ matrix.arch }} weed-worker-${{ matrix.arch }} + weed-vector-${{ matrix.arch }} # One job per (variant, platform) on a native runner, pushed by digest; # the merge job stitches the digests into one multi-arch tag. @@ -198,8 +206,17 @@ jobs: echo "Placed pre-built Rust worker for ${arch}" fi done + mkdir -p docker/weed-vector-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-vector-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}" + echo "Placed pre-built vector engine for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ ls -la docker/weed-worker-prebuilt/ + ls -la docker/weed-vector-prebuilt/ - name: Free Disk Space if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant diff --git a/.github/workflows/rust_binaries_release.yml b/.github/workflows/rust_binaries_release.yml index 3999c6df34..f6c6359ac5 100644 --- a/.github/workflows/rust_binaries_release.yml +++ b/.github/workflows/rust_binaries_release.yml @@ -188,6 +188,83 @@ jobs: weed-worker_${{ matrix.asset_suffix }}.tar.gz weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5 + # The vector engine behind the S3 Vectors API: Linux only, like the worker, + # whose lance dependency tree it shares. + build-rust-vector-linux: + permissions: + contents: write + runs-on: ubuntu-24.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-musl + asset_suffix: linux_amd64 + - target: aarch64-unknown-linux-musl + asset_suffix: linux_arm64 + + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up the musl target + uses: ./.github/actions/rust-musl-target + with: + target: ${{ matrix.target }} + + - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-vector/target/${{ matrix.target }}/release + key: rust-vector-release-${{ matrix.target }}-${{ hashFiles('seaweed-vector/Cargo.lock') }} + restore-keys: | + rust-vector-release-${{ matrix.target }}- + + - name: Use the vendored protoc + run: | + cd seaweed-vector + cargo fetch + version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock) + test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; } + protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1) + test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; } + echo "PROTOC=$protoc" >> "$GITHUB_ENV" + + - name: Build the vector engine + run: | + cd seaweed-vector + cargo build --release -p weed-vector-server --target ${{ matrix.target }} + + - name: Package binary + run: | + cp seaweed-vector/target/${{ matrix.target }}/release/weed-vector weed-vector + tar czf weed-vector_${{ matrix.asset_suffix }}.tar.gz weed-vector + rm weed-vector + md5sum weed-vector_${{ matrix.asset_suffix }}.tar.gz > weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5 + + - name: Upload release assets + if: startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v3 + with: + files: | + weed-vector_${{ matrix.asset_suffix }}.tar.gz + weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload artifacts + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + uses: actions/upload-artifact@v7 + with: + name: rust-vector-${{ matrix.asset_suffix }} + path: | + weed-vector_${{ matrix.asset_suffix }}.tar.gz + weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5 + build-rust-volume-darwin: permissions: contents: write diff --git a/docker/Dockerfile.go_build b/docker/Dockerfile.go_build index a2eb20568d..22e4674290 100644 --- a/docker/Dockerfile.go_build +++ b/docker/Dockerfile.go_build @@ -33,6 +33,7 @@ ARG TARGETARCH ARG TAGS COPY weed-volume-prebuilt/ /prebuilt/ COPY weed-worker-prebuilt/ /prebuilt-worker/ +COPY weed-vector-prebuilt/ /prebuilt-vector/ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume # seaweed-common is a path dependency of seaweed-volume that lives beside it, # so the source build below needs it in the same relative position. @@ -65,11 +66,19 @@ RUN if [ -f "/prebuilt-worker/weed-worker-${TARGETARCH}" ]; then \ echo "No pre-built Rust worker for ${TARGETARCH}" && \ touch /weed-worker; \ fi +# The vector engine carries the same lance tree, so it is pre-built or absent too. +RUN if [ -f "/prebuilt-vector/weed-vector-${TARGETARCH}" ]; then \ + echo "Using pre-built vector engine for ${TARGETARCH}" && \ + cp "/prebuilt-vector/weed-vector-${TARGETARCH}" /weed-vector; \ + else \ + echo "No pre-built vector engine for ${TARGETARCH}" && \ + touch /weed-vector; \ + fi # Pre-built binaries arrive via GitHub Actions artifacts, which drop the # executable bit, so the copied file is 0644 and exec fails with "Permission # denied". Restore it (no-op for the empty placeholders, which stay size 0). -RUN chmod 0755 /weed-volume /weed-worker +RUN chmod 0755 /weed-volume /weed-worker /weed-vector FROM alpine AS final LABEL author="Chris Lu" @@ -78,6 +87,8 @@ COPY --from=builder /go/bin/weed /usr/bin/ COPY --from=rust_builder /weed-volume /usr/bin/weed-volume # Same for the Rust maintenance worker, which serves Lance table buckets COPY --from=rust_builder /weed-worker /usr/bin/weed-worker +# And the vector engine behind the S3 Vectors API +COPY --from=rust_builder /weed-vector /usr/bin/weed-vector RUN mkdir -p /etc/seaweedfs COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 0bbd507871..8e6ee840c5 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -100,6 +100,19 @@ case "$1" in exec /usr/bin/weed-worker "$@" ;; + 'vector') + shift + if [ ! -s /usr/bin/weed-vector ]; then + echo "Error: the vector engine is not available on this platform ($(uname -m))." >&2 + exit 1 + fi + # The gateway dials the engine from another container, so not loopback. + if [ -z "$WEED_VECTOR_LISTEN" ] && ! isArgPassed "-listen" "$@"; then + set -- --listen 0.0.0.0:8090 "$@" + fi + exec /usr/bin/weed-vector "$@" + ;; + 'server') ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024" if isArgPassed "-volume.max" "$@"; then diff --git a/docker/weed-vector-prebuilt/.gitkeep b/docker/weed-vector-prebuilt/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/install.sh b/install.sh index 8bede247af..72574bf4cc 100755 --- a/install.sh +++ b/install.sh @@ -9,7 +9,7 @@ # curl -fsSL ... | bash -s -- --version 4.34 --dir /usr/local/bin # # Options: -# --component COMP Which binary to install: weed, volume-rust, worker-rust, all (default: weed) +# --component COMP Which binary to install: weed, volume-rust, worker-rust, vector, all (default: weed) # --version VER Release version tag (default: latest) # --large-disk Use large disk variant (5-byte offset, 8TB max volume) # --dir DIR Installation directory (default: /usr/local/bin) @@ -23,6 +23,7 @@ VERSION="" LARGE_DISK=false INSTALL_DIR="/usr/local/bin" WORKER_INSTALLED=false +VECTOR_INSTALLED=false # Colors (if terminal supports them) if [ -t 1 ]; then @@ -156,6 +157,12 @@ worker_asset_name() { echo "weed-worker_${os}_${arch}.tar.gz" } +# The vector engine behind the S3 Vectors API, built like the worker. +vector_asset_name() { + local os="$1" arch="$2" + echo "weed-vector_${os}_${arch}.tar.gz" +} + # Install a single component install_component() { local component="$1" os="$2" arch="$3" @@ -264,8 +271,38 @@ install_component() { ok "Installed weed-worker to ${INSTALL_DIR}/weed-worker" ;; + vector) + # Published for linux amd64/arm64 only, like the worker. + case "$os" in + linux) ;; + *) error "The vector engine is not available for ${os}. Supported: linux" ;; + esac + case "$arch" in + amd64|arm64) ;; + *) error "The vector engine is not available for ${arch}. Supported: amd64, arm64" ;; + esac + + asset_name="$(vector_asset_name "$os" "$arch")" + download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}" + download "$download_url" "${tmpdir}/${asset_name}" + + info "Extracting ${asset_name}..." + tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir" + + local vector_bin + vector_bin="$(find "$tmpdir" -name 'weed-vector' -type f | head -1)" + if [ -z "$vector_bin" ]; then + error "Could not find weed-vector binary in archive" + fi + + chmod +x "$vector_bin" + install_binary "$vector_bin" "weed-vector" + VECTOR_INSTALLED=true + ok "Installed weed-vector to ${INSTALL_DIR}/weed-vector" + ;; + *) - error "Unknown component: ${component}. Use: weed, volume-rust, worker-rust, all" + error "Unknown component: ${component}. Use: weed, volume-rust, worker-rust, vector, all" ;; esac @@ -327,6 +364,13 @@ main() { else install_component "worker-rust" "$os" "$arch" fi + if [ "$os" != "linux" ] || { [ "$arch" != "amd64" ] && [ "$arch" != "arm64" ]; }; then + warn "Skipping the vector engine: no build for ${os}/${arch}" + elif ! asset_exists "https://github.com/${REPO}/releases/download/${VERSION}/$(vector_asset_name "$os" "$arch")"; then + warn "Skipping the vector engine: ${VERSION} does not carry one" + else + install_component "vector" "$os" "$arch" + fi ;; *) install_component "$COMPONENT" "$os" "$arch" @@ -344,6 +388,9 @@ main() { if [ "$WORKER_INSTALLED" = true ]; then info " weed-worker: ${INSTALL_DIR}/weed-worker" fi + if [ "$VECTOR_INSTALLED" = true ]; then + info " weed-vector: ${INSTALL_DIR}/weed-vector" + fi echo "" info "Quick start:" info " weed master # Start master server" @@ -352,6 +399,9 @@ main() { if [ "$WORKER_INSTALLED" = true ]; then info " weed-worker --admin localhost:12535 # Start the Rust maintenance worker" fi + if [ "$VECTOR_INSTALLED" = true ]; then + info " weed-vector --help # The vector engine for S3 vector buckets" + fi } main