From 89d3016d5efe4f3ce8066b986f4585cab7622479 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 9 Nov 2026 16:14:42 -0800 Subject: [PATCH] vector: publish native macOS and Windows engines and installer support (#12523) * ci: build, test, and publish vector engines for macOS and Windows * installer: install native vector archives on macOS and Windows * ci: run desktop vector checks for shared Rust and protobuf changes * ci: publish vector releases from a separate job with write permissions * ci: build tagged vector releases without restoring caches * ci: propagate Windows bash path conversion failures --- .github/workflows/rust-vector-platforms.yml | 124 ++++++++++++++++++++ install.sh | 44 ++++--- test/installer/test_vector.py | 56 +++++++++ 3 files changed, 205 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/rust-vector-platforms.yml create mode 100644 test/installer/test_vector.py diff --git a/.github/workflows/rust-vector-platforms.yml b/.github/workflows/rust-vector-platforms.yml new file mode 100644 index 0000000000..8916a42d22 --- /dev/null +++ b/.github/workflows/rust-vector-platforms.yml @@ -0,0 +1,124 @@ +name: "rust: vector desktop platforms" +on: + pull_request: + paths: + - 'seaweed-vector/**' + - 'seaweed-worker/**' + - 'seaweed-common/**' + - 'weed/pb/**' + - '.github/workflows/rust-vector-platforms.yml' + - 'install.sh' + - 'test/installer/**' + push: + tags: ['*'] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: vector-platforms-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +jobs: + vector: + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - runner: macos-15-intel + suffix: darwin_amd64 + protoc: macos-x86_64 + - runner: macos-latest + suffix: darwin_arm64 + protoc: macos-aarch_64 + - runner: windows-latest + suffix: windows_amd64 + protoc: win32 + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + env: + CARGO_BUILD_JOBS: '2' + CARGO_PROFILE_TEST_DEBUG: '0' + SEAWEEDFS_COMMIT: ${{ github.sha }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + - uses: actions/cache@v6 + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-vector/target + key: vector-${{ matrix.suffix }}-${{ hashFiles('seaweed-vector/Cargo.lock') }} + restore-keys: vector-${{ matrix.suffix }}- + - name: Fetch dependencies + run: cargo fetch --locked --manifest-path seaweed-vector/Cargo.toml + - name: Select vendored protoc + env: + PROTOC_PACKAGE: protoc-bin-vendored-${{ matrix.protoc }} + run: | + python - <<'PY' + import os, pathlib, tomllib + lock=tomllib.loads(pathlib.Path('seaweed-vector/Cargo.lock').read_text()) + name=os.environ['PROTOC_PACKAGE'] + version=next(p['version'] for p in lock['package'] if p['name']==name) + cargo=pathlib.Path(os.environ.get('CARGO_HOME',pathlib.Path.home()/'.cargo')) + executable='protoc.exe' if os.name=='nt' else 'protoc' + path=next(cargo.glob(f'registry/src/*/{name}-{version}/bin/{executable}')) + with open(os.environ['GITHUB_ENV'],'a') as out: out.write(f'PROTOC={path}\n') + PY + - name: Build vector engine + run: cargo build --locked --release --manifest-path seaweed-vector/Cargo.toml -p weed-vector-server + - name: Test vector crates + run: cargo test --locked --release --manifest-path seaweed-vector/Cargo.toml --workspace + - name: Test installer + run: | + if [ "$RUNNER_OS" = "Windows" ]; then + BASH_BINARY="$(cygpath -w "$BASH")" + export BASH_BINARY + fi + python -m unittest discover -s test/installer + - name: Package engine + env: + ASSET_SUFFIX: ${{ matrix.suffix }} + run: | + python - <<'PY' + import hashlib, os, pathlib, tarfile, zipfile + suffix=os.environ['ASSET_SUFFIX'] + binary='weed-vector.exe' if os.name=='nt' else 'weed-vector' + source=pathlib.Path('seaweed-vector/target/release')/binary + asset=f'weed-vector_{suffix}'+('.zip' if os.name=='nt' else '.tar.gz') + if os.name=='nt': + with zipfile.ZipFile(asset,'w',zipfile.ZIP_DEFLATED) as archive: archive.write(source,binary) + else: + with tarfile.open(asset,'w:gz') as archive: archive.add(source,arcname=binary) + digest=hashlib.md5(pathlib.Path(asset).read_bytes()).hexdigest() + pathlib.Path(asset+'.md5').write_text(f'{digest} {asset}\n') + PY + - uses: actions/upload-artifact@v7 + with: + name: vector-${{ matrix.suffix }} + path: weed-vector_${{ matrix.suffix }}.* + release: + if: startsWith(github.ref, 'refs/tags/') + needs: vector + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/download-artifact@v8 + with: + pattern: vector-* + merge-multiple: true + - uses: softprops/action-gh-release@v3 + with: + files: weed-vector_* diff --git a/install.sh b/install.sh index 72574bf4cc..73ce1e92f3 100755 --- a/install.sh +++ b/install.sh @@ -160,7 +160,18 @@ worker_asset_name() { # The vector engine behind the S3 Vectors API, built like the worker. vector_asset_name() { local os="$1" arch="$2" - echo "weed-vector_${os}_${arch}.tar.gz" + if [ "$os" = "windows" ]; then + echo "weed-vector_${os}_${arch}.zip" + else + echo "weed-vector_${os}_${arch}.tar.gz" + fi +} + +vector_platform_supported() { + case "$1/$2" in + linux/amd64|linux/arm64|darwin/amd64|darwin/arm64|windows/amd64) return 0 ;; + *) return 1 ;; + esac } # Install a single component @@ -272,31 +283,26 @@ install_component() { ;; vector) - # Published for linux amd64/arm64 only, like the worker. - case "$os" in - linux) ;; - *) error "The vector engine is not available for ${os}. Supported: linux" ;; - esac - case "$arch" in - amd64|arm64) ;; - *) error "The vector engine is not available for ${arch}. Supported: amd64, arm64" ;; - esac + vector_platform_supported "$os" "$arch" || error "The vector engine is not available for ${os}/${arch}" asset_name="$(vector_asset_name "$os" "$arch")" download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}" download "$download_url" "${tmpdir}/${asset_name}" info "Extracting ${asset_name}..." - tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir" - - local vector_bin - vector_bin="$(find "$tmpdir" -name 'weed-vector' -type f | head -1)" - if [ -z "$vector_bin" ]; then - error "Could not find weed-vector binary in archive" + local vector_name="weed-vector" + if [ "$os" = "windows" ]; then + command -v unzip >/dev/null || error "unzip is required for Windows vector archives" + unzip -q "${tmpdir}/${asset_name}" -d "$tmpdir" + vector_name="weed-vector.exe" + else + tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir" fi - + local vector_bin + vector_bin="$(find "$tmpdir" -name "$vector_name" -type f | head -1)" + [ -n "$vector_bin" ] || error "Could not find $vector_name binary in archive" chmod +x "$vector_bin" - install_binary "$vector_bin" "weed-vector" + install_binary "$vector_bin" "$vector_name" VECTOR_INSTALLED=true ok "Installed weed-vector to ${INSTALL_DIR}/weed-vector" ;; @@ -364,7 +370,7 @@ main() { else install_component "worker-rust" "$os" "$arch" fi - if [ "$os" != "linux" ] || { [ "$arch" != "amd64" ] && [ "$arch" != "arm64" ]; }; then + if ! vector_platform_supported "$os" "$arch"; then warn "Skipping the vector engine: no build for ${os}/${arch}" elif ! asset_exists "https://github.com/${REPO}/releases/download/${VERSION}/$(vector_asset_name "$os" "$arch")"; then warn "Skipping the vector engine: ${VERSION} does not carry one" diff --git a/test/installer/test_vector.py b/test/installer/test_vector.py new file mode 100644 index 0000000000..ee2f2a7ec6 --- /dev/null +++ b/test/installer/test_vector.py @@ -0,0 +1,56 @@ +import io +import os +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest +import zipfile + +SOURCE = (Path(__file__).resolve().parents[2] / 'install.sh').read_text().rsplit('\nmain\n', 1)[0] +BASH = os.environ.get('BASH_BINARY', 'bash') + + +class VectorInstallerTest(unittest.TestCase): + def shell(self, code, **env): + return subprocess.run([BASH, '-c', SOURCE + '\n' + code], env=dict(os.environ, **env), + text=True, capture_output=True) + + def test_platforms_and_asset_names(self): + for system, arch, supported, suffix in [ + ('linux', 'amd64', True, 'tar.gz'), ('linux', 'arm64', True, 'tar.gz'), + ('darwin', 'amd64', True, 'tar.gz'), ('darwin', 'arm64', True, 'tar.gz'), + ('windows', 'amd64', True, 'zip'), ('windows', 'arm64', False, 'zip'), + ('freebsd', 'amd64', False, 'tar.gz'), + ]: + with self.subTest(system=system, arch=arch): + result = self.shell('vector_platform_supported "$SYSTEM" "$ARCH"; result=$?; ' + 'vector_asset_name "$SYSTEM" "$ARCH"; exit "$result"', SYSTEM=system, ARCH=arch) + if supported: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), f'weed-vector_{system}_{arch}.{suffix}') + else: + self.assertNotEqual(result.returncode, 0) + + def test_installs_archive_binary(self): + for system in ['linux', 'darwin', 'windows']: + with self.subTest(system=system), tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + name = 'weed-vector.exe' if system == 'windows' else 'weed-vector' + contents = b'vector engine fixture' + archive = directory / ('vector.zip' if system == 'windows' else 'vector.tar.gz') + if system == 'windows': + with zipfile.ZipFile(archive, 'w') as output: + output.writestr(name, contents) + else: + with tarfile.open(archive, 'w:gz') as output: + info = tarfile.TarInfo(name) + info.size = len(contents) + info.mode = 0o755 + output.addfile(info, io.BytesIO(contents)) + dest = directory / 'bin' + result = self.shell('download() { cp "$ARCHIVE" "$2"; }; VERSION=test; ' + 'INSTALL_DIR="$DEST"; install_component vector "$SYSTEM" amd64', + ARCHIVE=archive.as_posix(), DEST=dest.as_posix(), SYSTEM=system) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual((dest / name).read_bytes(), contents)