vector: publish native macOS and Windows engines and installer support (#12523)

* ci: build, test, and publish vector engines for macOS and Windows

* installer: install native vector archives on macOS and Windows

* ci: run desktop vector checks for shared Rust and protobuf changes

* ci: publish vector releases from a separate job with write permissions

* ci: build tagged vector releases without restoring caches

* ci: propagate Windows bash path conversion failures

.github/workflows/rust-vector-platforms.yml

@@ -0,0 +1,124 @@
+name: "rust: vector desktop platforms"
+on:
+  pull_request:
+    paths:
+      - 'seaweed-vector/**'
+      - 'seaweed-worker/**'
+      - 'seaweed-common/**'
+      - 'weed/pb/**'
+      - '.github/workflows/rust-vector-platforms.yml'
+      - 'install.sh'
+      - 'test/installer/**'
+  push:
+    tags: ['*']
+  workflow_dispatch:
+permissions:
+  contents: read
+concurrency:
+  group: vector-platforms-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+jobs:
+  vector:
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: macos-15-intel
+            suffix: darwin_amd64
+            protoc: macos-x86_64
+          - runner: macos-latest
+            suffix: darwin_arm64
+            protoc: macos-aarch_64
+          - runner: windows-latest
+            suffix: windows_amd64
+            protoc: win32
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 60
+    env:
+      CARGO_BUILD_JOBS: '2'
+      CARGO_PROFILE_TEST_DEBUG: '0'
+      SEAWEEDFS_COMMIT: ${{ github.sha }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.13'
+      - uses: actions/cache@v6
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-vector/target
+          key: vector-${{ matrix.suffix }}-${{ hashFiles('seaweed-vector/Cargo.lock') }}
+          restore-keys: vector-${{ matrix.suffix }}-
+      - name: Fetch dependencies
+        run: cargo fetch --locked --manifest-path seaweed-vector/Cargo.toml
+      - name: Select vendored protoc
+        env:
+          PROTOC_PACKAGE: protoc-bin-vendored-${{ matrix.protoc }}
+        run: |
+          python - <<'PY'
+          import os, pathlib, tomllib
+          lock=tomllib.loads(pathlib.Path('seaweed-vector/Cargo.lock').read_text())
+          name=os.environ['PROTOC_PACKAGE']
+          version=next(p['version'] for p in lock['package'] if p['name']==name)
+          cargo=pathlib.Path(os.environ.get('CARGO_HOME',pathlib.Path.home()/'.cargo'))
+          executable='protoc.exe' if os.name=='nt' else 'protoc'
+          path=next(cargo.glob(f'registry/src/*/{name}-{version}/bin/{executable}'))
+          with open(os.environ['GITHUB_ENV'],'a') as out: out.write(f'PROTOC={path}\n')
+          PY
+      - name: Build vector engine
+        run: cargo build --locked --release --manifest-path seaweed-vector/Cargo.toml -p weed-vector-server
+      - name: Test vector crates
+        run: cargo test --locked --release --manifest-path seaweed-vector/Cargo.toml --workspace
+      - name: Test installer
+        run: |
+          if [ "$RUNNER_OS" = "Windows" ]; then
+            BASH_BINARY="$(cygpath -w "$BASH")"
+            export BASH_BINARY
+          fi
+          python -m unittest discover -s test/installer
+      - name: Package engine
+        env:
+          ASSET_SUFFIX: ${{ matrix.suffix }}
+        run: |
+          python - <<'PY'
+          import hashlib, os, pathlib, tarfile, zipfile
+          suffix=os.environ['ASSET_SUFFIX']
+          binary='weed-vector.exe' if os.name=='nt' else 'weed-vector'
+          source=pathlib.Path('seaweed-vector/target/release')/binary
+          asset=f'weed-vector_{suffix}'+('.zip' if os.name=='nt' else '.tar.gz')
+          if os.name=='nt':
+              with zipfile.ZipFile(asset,'w',zipfile.ZIP_DEFLATED) as archive: archive.write(source,binary)
+          else:
+              with tarfile.open(asset,'w:gz') as archive: archive.add(source,arcname=binary)
+          digest=hashlib.md5(pathlib.Path(asset).read_bytes()).hexdigest()
+          pathlib.Path(asset+'.md5').write_text(f'{digest}  {asset}\n')
+          PY
+      - uses: actions/upload-artifact@v7
+        with:
+          name: vector-${{ matrix.suffix }}
+          path: weed-vector_${{ matrix.suffix }}.*
+  release:
+    if: startsWith(github.ref, 'refs/tags/')
+    needs: vector
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          pattern: vector-*
+          merge-multiple: true
+      - uses: softprops/action-gh-release@v3
+        with:
+          files: weed-vector_*

install.sh

@@ -160,7 +160,18 @@ worker_asset_name() {
 # The vector engine behind the S3 Vectors API, built like the worker.
 vector_asset_name() {
     local os="$1" arch="$2"
-    echo "weed-vector_${os}_${arch}.tar.gz"
+    if [ "$os" = "windows" ]; then
+        echo "weed-vector_${os}_${arch}.zip"
+    else
+        echo "weed-vector_${os}_${arch}.tar.gz"
+    fi
+}
+
+vector_platform_supported() {
+    case "$1/$2" in
+        linux/amd64|linux/arm64|darwin/amd64|darwin/arm64|windows/amd64) return 0 ;;
+        *) return 1 ;;
+    esac
 }
 
 # Install a single component
@@ -272,31 +283,26 @@ install_component() {
             ;;
 
         vector)
-            # Published for linux amd64/arm64 only, like the worker.
-            case "$os" in
-                linux) ;;
-                *) error "The vector engine is not available for ${os}. Supported: linux" ;;
-            esac
-            case "$arch" in
-                amd64|arm64) ;;
-                *) error "The vector engine is not available for ${arch}. Supported: amd64, arm64" ;;
-            esac
+            vector_platform_supported "$os" "$arch" || error "The vector engine is not available for ${os}/${arch}"
 
             asset_name="$(vector_asset_name "$os" "$arch")"
             download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
             download "$download_url" "${tmpdir}/${asset_name}"
 
             info "Extracting ${asset_name}..."
-            tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
-
-            local vector_bin
-            vector_bin="$(find "$tmpdir" -name 'weed-vector' -type f | head -1)"
-            if [ -z "$vector_bin" ]; then
-                error "Could not find weed-vector binary in archive"
+            local vector_name="weed-vector"
+            if [ "$os" = "windows" ]; then
+                command -v unzip >/dev/null || error "unzip is required for Windows vector archives"
+                unzip -q "${tmpdir}/${asset_name}" -d "$tmpdir"
+                vector_name="weed-vector.exe"
+            else
+                tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
             fi
-
+            local vector_bin
+            vector_bin="$(find "$tmpdir" -name "$vector_name" -type f | head -1)"
+            [ -n "$vector_bin" ] || error "Could not find $vector_name binary in archive"
             chmod +x "$vector_bin"
-            install_binary "$vector_bin" "weed-vector"
+            install_binary "$vector_bin" "$vector_name"
             VECTOR_INSTALLED=true
             ok "Installed weed-vector to ${INSTALL_DIR}/weed-vector"
             ;;
@@ -364,7 +370,7 @@ main() {
             else
                 install_component "worker-rust" "$os" "$arch"
             fi
-            if [ "$os" != "linux" ] || { [ "$arch" != "amd64" ] && [ "$arch" != "arm64" ]; }; then
+            if ! vector_platform_supported "$os" "$arch"; then
                 warn "Skipping the vector engine: no build for ${os}/${arch}"
             elif ! asset_exists "https://github.com/${REPO}/releases/download/${VERSION}/$(vector_asset_name "$os" "$arch")"; then
                 warn "Skipping the vector engine: ${VERSION} does not carry one"

test/installer/test_vector.py

@@ -0,0 +1,56 @@
+import io
+import os
+from pathlib import Path
+import subprocess
+import tarfile
+import tempfile
+import unittest
+import zipfile
+
+SOURCE = (Path(__file__).resolve().parents[2] / 'install.sh').read_text().rsplit('\nmain\n', 1)[0]
+BASH = os.environ.get('BASH_BINARY', 'bash')
+
+
+class VectorInstallerTest(unittest.TestCase):
+    def shell(self, code, **env):
+        return subprocess.run([BASH, '-c', SOURCE + '\n' + code], env=dict(os.environ, **env),
+                              text=True, capture_output=True)
+
+    def test_platforms_and_asset_names(self):
+        for system, arch, supported, suffix in [
+            ('linux', 'amd64', True, 'tar.gz'), ('linux', 'arm64', True, 'tar.gz'),
+            ('darwin', 'amd64', True, 'tar.gz'), ('darwin', 'arm64', True, 'tar.gz'),
+            ('windows', 'amd64', True, 'zip'), ('windows', 'arm64', False, 'zip'),
+            ('freebsd', 'amd64', False, 'tar.gz'),
+        ]:
+            with self.subTest(system=system, arch=arch):
+                result = self.shell('vector_platform_supported "$SYSTEM" "$ARCH"; result=$?; '
+                                    'vector_asset_name "$SYSTEM" "$ARCH"; exit "$result"', SYSTEM=system, ARCH=arch)
+                if supported:
+                    self.assertEqual(result.returncode, 0, result.stderr)
+                    self.assertEqual(result.stdout.strip(), f'weed-vector_{system}_{arch}.{suffix}')
+                else:
+                    self.assertNotEqual(result.returncode, 0)
+
+    def test_installs_archive_binary(self):
+        for system in ['linux', 'darwin', 'windows']:
+            with self.subTest(system=system), tempfile.TemporaryDirectory() as directory:
+                directory = Path(directory)
+                name = 'weed-vector.exe' if system == 'windows' else 'weed-vector'
+                contents = b'vector engine fixture'
+                archive = directory / ('vector.zip' if system == 'windows' else 'vector.tar.gz')
+                if system == 'windows':
+                    with zipfile.ZipFile(archive, 'w') as output:
+                        output.writestr(name, contents)
+                else:
+                    with tarfile.open(archive, 'w:gz') as output:
+                        info = tarfile.TarInfo(name)
+                        info.size = len(contents)
+                        info.mode = 0o755
+                        output.addfile(info, io.BytesIO(contents))
+                dest = directory / 'bin'
+                result = self.shell('download() { cp "$ARCHIVE" "$2"; }; VERSION=test; '
+                                    'INSTALL_DIR="$DEST"; install_component vector "$SYSTEM" amd64',
+                                    ARCHIVE=archive.as_posix(), DEST=dest.as_posix(), SYSTEM=system)
+                self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+                self.assertEqual((dest / name).read_bytes(), contents)