From 54777d7fb91c0ad9d63ae3d7c521f193f0c8c638 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 1 Nov 2026 18:56:09 -0800 Subject: [PATCH] vector: build weed-vector without a system protoc (#12317) * vector: build without a system protoc lance's build scripts run $PROTOC and fail without one. Point it at the protoc that protoc-bin-vendored ships, at the version in Cargo.lock, so building inside seaweed-vector/ needs only the Rust toolchain. An explicit PROTOC still wins, and a registry without the crate falls back to PATH. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3vectors): build weed-vector inside its workspace Cargo reads seaweed-vector/.cargo/config.toml only when it runs there, so build from that directory to pick up the vendored protoc. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * ci(vector): drop the PROTOC step the workspace config replaces Builds run inside seaweed-vector/, where the cargo config already points lance at the vendored protoc. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * test(s3vectors): resolve CARGO_TARGET_DIR before changing directory A relative CARGO_TARGET_DIR given to make would point cargo, which now runs in seaweed-vector/, and the cp after it at different directories. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * vector: pick the vendored protoc on every Linux host it ships for protoc-bin-vendored also carries ppc64le, s390x and x86_32 binaries. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/rust-vector-tests.yml | 22 ---------------------- seaweed-vector/.cargo/config.toml | 6 ++++++ seaweed-vector/.cargo/protoc | 21 +++++++++++++++++++++ test/s3vectors/Makefile | 3 ++- 4 files changed, 29 insertions(+), 23 deletions(-) create mode 100644 seaweed-vector/.cargo/config.toml create mode 100755 seaweed-vector/.cargo/protoc diff --git a/.github/workflows/rust-vector-tests.yml b/.github/workflows/rust-vector-tests.yml index 1776ae513a..431b0d9d99 100644 --- a/.github/workflows/rust-vector-tests.yml +++ b/.github/workflows/rust-vector-tests.yml @@ -68,19 +68,6 @@ jobs: restore-keys: | rust-vector-${{ steps.toolchain.outputs.fingerprint }}- - # lance's build scripts compile their own protos and look for a protoc. - # Point them at the one protoc-bin-vendored ships, so no job depends on a - # system package and every build sees the same version. - - name: Use the vendored protoc - run: | - cd seaweed-vector - cargo fetch - version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock) - test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; } - protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1) - test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; } - echo "PROTOC=$protoc" >> "$GITHUB_ENV" - - name: Build the vector crates run: cd seaweed-vector && cargo build --release @@ -125,15 +112,6 @@ jobs: key: vectors-e2e-${{ runner.os }}-${{ runner.arch }}-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-vector/Cargo.lock', 'seaweed-volume/Cargo.lock') }} restore-keys: | vectors-e2e-${{ runner.os }}-${{ runner.arch }}-${{ steps.toolchain.outputs.fingerprint }}- - - name: Use the vendored protoc - run: | - cd seaweed-vector - cargo fetch - version=$(awk '/^name = "protoc-bin-vendored-linux-x86_64"$/{found=1; next} found && /^version = /{gsub(/"/,"",$3); print $3; exit}' Cargo.lock) - test -n "$version" || { echo "protoc-bin-vendored-linux-x86_64 is not in Cargo.lock" >&2; exit 1; } - protoc=$(find ~/.cargo/registry/src -path "*protoc-bin-vendored-linux-x86_64-$version/bin/protoc" | head -1) - test -x "$protoc" || { echo "no vendored protoc $version in the registry" >&2; exit 1; } - echo "PROTOC=$protoc" >> "$GITHUB_ENV" - name: Test Go vector gateway run: | go test ./weed/s3api/s3vectors -count=1 diff --git a/seaweed-vector/.cargo/config.toml b/seaweed-vector/.cargo/config.toml new file mode 100644 index 0000000000..d3b5a7e240 --- /dev/null +++ b/seaweed-vector/.cargo/config.toml @@ -0,0 +1,6 @@ +# lance's build scripts run $PROTOC. Point them at the protoc that +# protoc-bin-vendored ships, as this workspace's own build script does, so a +# build needs no system protoc. An explicit PROTOC still wins. Cargo reads this +# file only when it runs inside seaweed-vector/. +[env] +PROTOC = { value = ".cargo/protoc", relative = true } diff --git a/seaweed-vector/.cargo/protoc b/seaweed-vector/.cargo/protoc new file mode 100755 index 0000000000..3f1717c478 --- /dev/null +++ b/seaweed-vector/.cargo/protoc @@ -0,0 +1,21 @@ +#!/bin/sh +# Runs the protoc of the protoc-bin-vendored version in Cargo.lock, or the one +# on PATH when the registry does not hold it. +case "$(uname -s)-$(uname -m)" in +Linux-x86_64) platform=linux-x86_64 ;; +Linux-aarch64) platform=linux-aarch_64 ;; +Linux-ppc64le) platform=linux-ppcle_64 ;; +Linux-s390x) platform=linux-s390_64 ;; +Linux-i?86) platform=linux-x86_32 ;; +Darwin-x86_64) platform=macos-x86_64 ;; +Darwin-arm64) platform=macos-aarch_64 ;; +*) exec protoc "$@" ;; +esac +crate="protoc-bin-vendored-$platform" +version=$(awk -v name="\"$crate\"" '$1 == "name" && $3 == name { found = 1; next } found && $1 == "version" { gsub(/"/, "", $3); print $3; exit }' "$(dirname "$0")/../Cargo.lock") +for protoc in "${CARGO_HOME:-$HOME/.cargo}"/registry/src/*/"$crate-$version"/bin/protoc; do + if [ -x "$protoc" ]; then + exec "$protoc" "$@" + fi +done +exec protoc "$@" diff --git a/test/s3vectors/Makefile b/test/s3vectors/Makefile index b5d301243a..5a390bb950 100644 --- a/test/s3vectors/Makefile +++ b/test/s3vectors/Makefile @@ -7,6 +7,7 @@ endif VOLUME := ./weed-volume CARGO_PROFILE ?= release CARGO_TARGET_DIR ?= $(abspath ../../seaweed-vector/target) +override CARGO_TARGET_DIR := $(abspath $(CARGO_TARGET_DIR)) CARGO_OUTPUT := $(CARGO_TARGET_DIR)/$(if $(filter dev,$(CARGO_PROFILE)),debug,$(CARGO_PROFILE)) export CARGO_TARGET_DIR @@ -18,7 +19,7 @@ $(WEED): FORCE go build -o $(WEED) ../../weed $(VEC): FORCE - cargo build --locked --profile $(CARGO_PROFILE) --manifest-path ../../seaweed-vector/Cargo.toml -p weed-vector-server + cd ../../seaweed-vector && cargo build --locked --profile $(CARGO_PROFILE) -p weed-vector-server cp $(CARGO_OUTPUT)/weed-vector $(VEC) ifneq ($(WORKER),)