Ship weed-vector in release assets and Docker images (#12322)

* ci(release): publish weed-vector binaries

Tagged releases now carry weed-vector_linux_{amd64,arm64}.tar.gz, static
musl builds made the same way as weed-worker's.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK

* docker: carry weed-vector in the image

The image takes a pre-built weed-vector the way it takes weed-worker, or an
empty placeholder where none was built. `vector` runs it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK

* ci(docker): build weed-vector for the images

The container workflows cross-compile weed-vector for amd64 and arm64
beside weed-worker and place it in the Docker context.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK

* install.sh: add the vector component

`--component vector` installs weed-vector from a release, and `all` adds it
when the release carries one, as it does for the worker.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK

* ci(docker): skip weed-vector for source refs that predate it

The manual latest-image rebuild accepts any source_ref; one without the
engine's server crate now builds its image without weed-vector instead of
failing the Rust job.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK

* docker: have the vector engine listen beyond loopback

weed-vector listens on 127.0.0.1 by default, which a gateway in another
container cannot reach. `vector` now adds --listen 0.0.0.0:8090 unless a
listen address is given by flag or WEED_VECTOR_LISTEN. The engine registers
its container address for a wildcard listen.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

.github/workflows/container_dev.yml

@@ -8,6 +8,7 @@ on:
       - 'seaweed-volume/**'
       - 'seaweed-common/**'
       - 'seaweed-worker/**'
+      - 'seaweed-vector/**'
       - 'docker/**'
       - 'go.mod'
       - 'go.sum'
@@ -50,7 +51,8 @@ jobs:
             ~/.cargo/git
             seaweed-volume/target
             seaweed-worker/target/${{ matrix.target }}/release
-          key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
+            seaweed-vector/target/${{ matrix.target }}/release
+          key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }}
           restore-keys: |
             rust-docker-dev-${{ matrix.target }}-
 
@@ -83,6 +85,12 @@ jobs:
           cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
           cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
 
+      - name: Build the vector engine
+        run: |
+          cd seaweed-vector
+          cargo build --release -p weed-vector-server --target ${{ matrix.target }}
+          cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }}
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v7
         with:
@@ -90,6 +98,7 @@ jobs:
           path: |
             weed-volume-normal-${{ matrix.arch }}
             weed-worker-${{ matrix.arch }}
+            weed-vector-${{ matrix.arch }}
 
   build-dev-containers:
     needs: [build-rust-binaries]
@@ -127,8 +136,17 @@ jobs:
               echo "Placed pre-built Rust worker for ${arch}"
             fi
           done
+          mkdir -p docker/weed-vector-prebuilt
+          for arch in amd64 arm64; do
+            src="./rust-bins/weed-vector-${arch}"
+            if [ -f "$src" ]; then
+              cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
+              echo "Placed pre-built vector engine for ${arch}"
+            fi
+          done
           ls -la docker/weed-volume-prebuilt/
           ls -la docker/weed-worker-prebuilt/
+          ls -la docker/weed-vector-prebuilt/
 
       - name: Docker meta
         id: docker_meta

.github/workflows/container_latest.yml

@@ -96,7 +96,8 @@ jobs:
             ~/.cargo/git
             seaweed-volume/target
             seaweed-worker/target/${{ matrix.target }}/release
-          key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
+            seaweed-vector/target/${{ matrix.target }}/release
+          key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }}
           restore-keys: |
             rust-docker-${{ matrix.target }}-
 
@@ -137,6 +138,17 @@ jobs:
           cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
           cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
 
+      - name: Build the vector engine
+        run: |
+          # An older source_ref predates the engine; its image goes without one.
+          if [ ! -f seaweed-vector/crates/server/Cargo.toml ]; then
+            echo "No vector engine in this source ref"
+            exit 0
+          fi
+          cd seaweed-vector
+          cargo build --release -p weed-vector-server --target ${{ matrix.target }}
+          cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }}
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v7
         with:
@@ -145,6 +157,7 @@ jobs:
             weed-volume-large-disk-${{ matrix.arch }}
             weed-volume-normal-${{ matrix.arch }}
             weed-worker-${{ matrix.arch }}
+            weed-vector-${{ matrix.arch }}
 
   build:
     needs: [setup, build-rust-binaries]
@@ -217,8 +230,17 @@ jobs:
               echo "Placed pre-built Rust worker for ${arch}"
             fi
           done
+          mkdir -p docker/weed-vector-prebuilt
+          for arch in amd64 arm64; do
+            src="./rust-bins/weed-vector-${arch}"
+            if [ -f "$src" ]; then
+              cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
+              echo "Placed pre-built vector engine for ${arch}"
+            fi
+          done
           ls -la docker/weed-volume-prebuilt/
           ls -la docker/weed-worker-prebuilt/
+          ls -la docker/weed-vector-prebuilt/
 
       - name: Docker meta
         id: docker_meta
@@ -342,8 +364,17 @@ jobs:
               echo "Placed pre-built Rust worker for ${arch}"
             fi
           done
+          mkdir -p docker/weed-vector-prebuilt
+          for arch in amd64 arm64; do
+            src="./rust-bins/weed-vector-${arch}"
+            if [ -f "$src" ]; then
+              cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
+              echo "Placed pre-built vector engine for ${arch}"
+            fi
+          done
           ls -la docker/weed-volume-prebuilt/
           ls -la docker/weed-worker-prebuilt/
+          ls -la docker/weed-vector-prebuilt/
       - name: Create BuildKit config for local scan build
         if: needs.setup.outputs.publish != 'true'
         run: |

.github/workflows/container_release_unified.yml

@@ -43,9 +43,9 @@ concurrency:
 jobs:
 
   # ── Pre-build the Rust binaries natively ────────────────────────────
-  # The volume server and the Rust maintenance worker, cross-compiled for
-  # amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into
-  # ~15 minutes of native compilation.
+  # The volume server, the Rust maintenance worker and the vector engine,
+  # cross-compiled for amd64 and arm64 without QEMU, turning a 5-hour emulated
+  # cargo build into ~15 minutes of native compilation.
   build-rust-binaries:
     runs-on: ubuntu-24.04
     strategy:
@@ -73,7 +73,8 @@ jobs:
             ~/.cargo/git
             seaweed-volume/target
             seaweed-worker/target/${{ matrix.target }}/release
-          key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }}
+            seaweed-vector/target/${{ matrix.target }}/release
+          key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock', 'seaweed-vector/Cargo.lock') }}
           restore-keys: |
             rust-docker-${{ matrix.target }}-
 
@@ -114,6 +115,12 @@ jobs:
           cargo build --release -p weed-lance-worker --target ${{ matrix.target }}
           cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }}
 
+      - name: Build the vector engine
+        run: |
+          cd seaweed-vector
+          cargo build --release -p weed-vector-server --target ${{ matrix.target }}
+          cp target/${{ matrix.target }}/release/weed-vector ../weed-vector-${{ matrix.arch }}
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v7
         with:
@@ -122,6 +129,7 @@ jobs:
             weed-volume-large-disk-${{ matrix.arch }}
             weed-volume-normal-${{ matrix.arch }}
             weed-worker-${{ matrix.arch }}
+            weed-vector-${{ matrix.arch }}
 
   # One job per (variant, platform) on a native runner, pushed by digest;
   # the merge job stitches the digests into one multi-arch tag.
@@ -198,8 +206,17 @@ jobs:
               echo "Placed pre-built Rust worker for ${arch}"
             fi
           done
+          mkdir -p docker/weed-vector-prebuilt
+          for arch in amd64 arm64; do
+            src="./rust-bins/weed-vector-${arch}"
+            if [ -f "$src" ]; then
+              cp "$src" "docker/weed-vector-prebuilt/weed-vector-${arch}"
+              echo "Placed pre-built vector engine for ${arch}"
+            fi
+          done
           ls -la docker/weed-volume-prebuilt/
           ls -la docker/weed-worker-prebuilt/
+          ls -la docker/weed-vector-prebuilt/
 
       - name: Free Disk Space
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant

.github/workflows/rust_binaries_release.yml

@@ -188,6 +188,83 @@ jobs:
             weed-worker_${{ matrix.asset_suffix }}.tar.gz
             weed-worker_${{ matrix.asset_suffix }}.tar.gz.md5
 
+  # The vector engine behind the S3 Vectors API: Linux only, like the worker,
+  # whose lance dependency tree it shares.
+  build-rust-vector-linux:
+    permissions:
+      contents: write
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        include:
+          - target: x86_64-unknown-linux-musl
+            asset_suffix: linux_amd64
+          - target: aarch64-unknown-linux-musl
+            asset_suffix: linux_arm64
+
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Set up the musl target
+        uses: ./.github/actions/rust-musl-target
+        with:
+          target: ${{ matrix.target }}
+
+      - name: Cache cargo registry and target
+        if: runner.environment == 'github-hosted'
+        uses: actions/cache@v6
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-vector/target/${{ matrix.target }}/release
+          key: rust-vector-release-${{ matrix.target }}-${{ hashFiles('seaweed-vector/Cargo.lock') }}
+          restore-keys: |
+            rust-vector-release-${{ matrix.target }}-
+
+      - name: Use the vendored protoc
+        run: |
+          cd seaweed-vector
+          cargo fetch
+          version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock)
+          test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; }
+          protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1)
+          test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; }
+          echo "PROTOC=$protoc" >> "$GITHUB_ENV"
+
+      - name: Build the vector engine
+        run: |
+          cd seaweed-vector
+          cargo build --release -p weed-vector-server --target ${{ matrix.target }}
+
+      - name: Package binary
+        run: |
+          cp seaweed-vector/target/${{ matrix.target }}/release/weed-vector weed-vector
+          tar czf weed-vector_${{ matrix.asset_suffix }}.tar.gz weed-vector
+          rm weed-vector
+          md5sum weed-vector_${{ matrix.asset_suffix }}.tar.gz > weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5
+
+      - name: Upload release assets
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: softprops/action-gh-release@v3
+        with:
+          files: |
+            weed-vector_${{ matrix.asset_suffix }}.tar.gz
+            weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload artifacts
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: rust-vector-${{ matrix.asset_suffix }}
+          path: |
+            weed-vector_${{ matrix.asset_suffix }}.tar.gz
+            weed-vector_${{ matrix.asset_suffix }}.tar.gz.md5
+
   build-rust-volume-darwin:
     permissions:
       contents: write

docker/Dockerfile.go_build

@@ -33,6 +33,7 @@ ARG TARGETARCH
 ARG TAGS
 COPY weed-volume-prebuilt/ /prebuilt/
 COPY weed-worker-prebuilt/ /prebuilt-worker/
+COPY weed-vector-prebuilt/ /prebuilt-vector/
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
 # seaweed-common is a path dependency of seaweed-volume that lives beside it,
 # so the source build below needs it in the same relative position.
@@ -65,11 +66,19 @@ RUN if [ -f "/prebuilt-worker/weed-worker-${TARGETARCH}" ]; then \
       echo "No pre-built Rust worker for ${TARGETARCH}" && \
       touch /weed-worker; \
     fi
+# The vector engine carries the same lance tree, so it is pre-built or absent too.
+RUN if [ -f "/prebuilt-vector/weed-vector-${TARGETARCH}" ]; then \
+      echo "Using pre-built vector engine for ${TARGETARCH}" && \
+      cp "/prebuilt-vector/weed-vector-${TARGETARCH}" /weed-vector; \
+    else \
+      echo "No pre-built vector engine for ${TARGETARCH}" && \
+      touch /weed-vector; \
+    fi
 
 # Pre-built binaries arrive via GitHub Actions artifacts, which drop the
 # executable bit, so the copied file is 0644 and exec fails with "Permission
 # denied". Restore it (no-op for the empty placeholders, which stay size 0).
-RUN chmod 0755 /weed-volume /weed-worker
+RUN chmod 0755 /weed-volume /weed-worker /weed-vector
 
 FROM alpine AS final
 LABEL author="Chris Lu"
@@ -78,6 +87,8 @@ COPY --from=builder /go/bin/weed /usr/bin/
 COPY --from=rust_builder /weed-volume /usr/bin/weed-volume
 # Same for the Rust maintenance worker, which serves Lance table buckets
 COPY --from=rust_builder /weed-worker /usr/bin/weed-worker
+# And the vector engine behind the S3 Vectors API
+COPY --from=rust_builder /weed-vector /usr/bin/weed-vector
 RUN mkdir -p /etc/seaweedfs
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh

docker/entrypoint.sh

@@ -100,6 +100,19 @@ case "$1" in
   	exec /usr/bin/weed-worker "$@"
 	;;
 
+  'vector')
+  	shift
+  	if [ ! -s /usr/bin/weed-vector ]; then
+  	  echo "Error: the vector engine is not available on this platform ($(uname -m))." >&2
+  	  exit 1
+  	fi
+  	# The gateway dials the engine from another container, so not loopback.
+  	if [ -z "$WEED_VECTOR_LISTEN" ] && ! isArgPassed "-listen" "$@"; then
+  	  set -- --listen 0.0.0.0:8090 "$@"
+  	fi
+  	exec /usr/bin/weed-vector "$@"
+	;;
+
   'server')
   	ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024"
   	if isArgPassed "-volume.max" "$@"; then

install.sh

@@ -9,7 +9,7 @@
 #   curl -fsSL ... | bash -s -- --version 4.34 --dir /usr/local/bin
 #
 # Options:
-#   --component COMP   Which binary to install: weed, volume-rust, worker-rust, all (default: weed)
+#   --component COMP   Which binary to install: weed, volume-rust, worker-rust, vector, all (default: weed)
 #   --version VER      Release version tag (default: latest)
 #   --large-disk       Use large disk variant (5-byte offset, 8TB max volume)
 #   --dir DIR          Installation directory (default: /usr/local/bin)
@@ -23,6 +23,7 @@ VERSION=""
 LARGE_DISK=false
 INSTALL_DIR="/usr/local/bin"
 WORKER_INSTALLED=false
+VECTOR_INSTALLED=false
 
 # Colors (if terminal supports them)
 if [ -t 1 ]; then
@@ -156,6 +157,12 @@ worker_asset_name() {
     echo "weed-worker_${os}_${arch}.tar.gz"
 }
 
+# The vector engine behind the S3 Vectors API, built like the worker.
+vector_asset_name() {
+    local os="$1" arch="$2"
+    echo "weed-vector_${os}_${arch}.tar.gz"
+}
+
 # Install a single component
 install_component() {
     local component="$1" os="$2" arch="$3"
@@ -264,8 +271,38 @@ install_component() {
             ok "Installed weed-worker to ${INSTALL_DIR}/weed-worker"
             ;;
 
+        vector)
+            # Published for linux amd64/arm64 only, like the worker.
+            case "$os" in
+                linux) ;;
+                *) error "The vector engine is not available for ${os}. Supported: linux" ;;
+            esac
+            case "$arch" in
+                amd64|arm64) ;;
+                *) error "The vector engine is not available for ${arch}. Supported: amd64, arm64" ;;
+            esac
+
+            asset_name="$(vector_asset_name "$os" "$arch")"
+            download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
+            download "$download_url" "${tmpdir}/${asset_name}"
+
+            info "Extracting ${asset_name}..."
+            tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
+
+            local vector_bin
+            vector_bin="$(find "$tmpdir" -name 'weed-vector' -type f | head -1)"
+            if [ -z "$vector_bin" ]; then
+                error "Could not find weed-vector binary in archive"
+            fi
+
+            chmod +x "$vector_bin"
+            install_binary "$vector_bin" "weed-vector"
+            VECTOR_INSTALLED=true
+            ok "Installed weed-vector to ${INSTALL_DIR}/weed-vector"
+            ;;
+
         *)
-            error "Unknown component: ${component}. Use: weed, volume-rust, worker-rust, all"
+            error "Unknown component: ${component}. Use: weed, volume-rust, worker-rust, vector, all"
             ;;
     esac
 
@@ -327,6 +364,13 @@ main() {
             else
                 install_component "worker-rust" "$os" "$arch"
             fi
+            if [ "$os" != "linux" ] || { [ "$arch" != "amd64" ] && [ "$arch" != "arm64" ]; }; then
+                warn "Skipping the vector engine: no build for ${os}/${arch}"
+            elif ! asset_exists "https://github.com/${REPO}/releases/download/${VERSION}/$(vector_asset_name "$os" "$arch")"; then
+                warn "Skipping the vector engine: ${VERSION} does not carry one"
+            else
+                install_component "vector" "$os" "$arch"
+            fi
             ;;
         *)
             install_component "$COMPONENT" "$os" "$arch"
@@ -344,6 +388,9 @@ main() {
     if [ "$WORKER_INSTALLED" = true ]; then
         info "  weed-worker:     ${INSTALL_DIR}/weed-worker"
     fi
+    if [ "$VECTOR_INSTALLED" = true ]; then
+        info "  weed-vector:     ${INSTALL_DIR}/weed-vector"
+    fi
     echo ""
     info "Quick start:"
     info "  weed master                          # Start master server"
@@ -352,6 +399,9 @@ main() {
     if [ "$WORKER_INSTALLED" = true ]; then
         info "  weed-worker --admin localhost:12535  # Start the Rust maintenance worker"
     fi
+    if [ "$VECTOR_INSTALLED" = true ]; then
+        info "  weed-vector --help                   # The vector engine for S3 vector buckets"
+    fi
 }
 
 main