From 2376f63a499bd1fd9be18172dad3cda1e08e6832 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 22 Aug 2026 01:38:23 -0700 Subject: [PATCH] ci: build the Rust worker for the container images The same native cross-compile the volume server uses, so the release, latest and dev images all carry it on amd64 and arm64. The artifact holds both binaries now, so it is named for that rather than for the volume server. Only the release directory each job builds is cached: with a debug profile beside it the worker's target/ reaches 24GB, against a 10GB cache budget. Claude-Session: https://claude.ai/code/session_01Rkp1Mw5E89Jp6dzJFYiMrm --- .github/workflows/container_dev.yml | 29 ++++++++++++--- .github/workflows/container_latest.yml | 36 ++++++++++++++++--- .../workflows/container_release_unified.yml | 30 ++++++++++++---- 3 files changed, 79 insertions(+), 16 deletions(-) diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index 11b339bc6..19ddf9c51 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -6,6 +6,7 @@ on: paths: - 'weed/**' - 'seaweed-volume/**' + - 'seaweed-worker/**' - 'docker/**' - 'go.mod' - 'go.sum' @@ -16,7 +17,7 @@ permissions: jobs: - # ── Pre-build Rust volume server binaries natively ────────────────── + # ── Pre-build the Rust binaries natively ──────────────────────────── build-rust-binaries: runs-on: ubuntu-22.04 strategy: @@ -55,7 +56,8 @@ jobs: ~/.cargo/registry ~/.cargo/git seaweed-volume/target - key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + seaweed-worker/target/${{ matrix.target }}/release + key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} restore-keys: | rust-docker-dev-${{ matrix.target }}- @@ -67,11 +69,19 @@ jobs: cargo build --release --target ${{ matrix.target }} --no-default-features cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }} + - name: Build the Rust maintenance worker + run: | + cd seaweed-worker + cargo build --release -p weed-lance-worker --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: - name: rust-volume-${{ matrix.arch }} - path: weed-volume-normal-${{ matrix.arch }} + name: rust-bins-${{ matrix.arch }} + path: | + weed-volume-normal-${{ matrix.arch }} + weed-worker-${{ matrix.arch }} build-dev-containers: needs: [build-rust-binaries] @@ -84,7 +94,7 @@ jobs: - name: Download pre-built Rust binaries uses: actions/download-artifact@v8 with: - pattern: rust-volume-* + pattern: rust-bins-* merge-multiple: true path: ./rust-bins @@ -98,7 +108,16 @@ jobs: echo "Placed pre-built Rust binary for ${arch}" fi done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index a82755c2e..55f2bd7f0 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -59,7 +59,7 @@ jobs: echo "publish=true" >> "$GITHUB_OUTPUT" fi - # ── Pre-build Rust volume server binaries natively ────────────────── + # ── Pre-build the Rust binaries natively ──────────────────────────── build-rust-binaries: runs-on: ubuntu-22.04 strategy: @@ -100,7 +100,8 @@ jobs: ~/.cargo/registry ~/.cargo/git seaweed-volume/target - key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + seaweed-worker/target/${{ matrix.target }}/release + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} restore-keys: | rust-docker-${{ matrix.target }}- @@ -120,13 +121,20 @@ jobs: cargo build --release --target ${{ matrix.target }} --no-default-features cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }} + - name: Build the Rust maintenance worker + run: | + cd seaweed-worker + cargo build --release -p weed-lance-worker --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: - name: rust-volume-${{ matrix.arch }} + name: rust-bins-${{ matrix.arch }} path: | weed-volume-large-disk-${{ matrix.arch }} weed-volume-normal-${{ matrix.arch }} + weed-worker-${{ matrix.arch }} build: needs: [setup, build-rust-binaries] @@ -174,7 +182,7 @@ jobs: - name: Download pre-built Rust binaries uses: actions/download-artifact@v8 with: - pattern: rust-volume-* + pattern: rust-bins-* merge-multiple: true path: ./rust-bins @@ -188,7 +196,16 @@ jobs: echo "Placed pre-built Rust binary for ${arch}" fi done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ - name: Docker meta id: docker_meta @@ -286,7 +303,7 @@ jobs: if: needs.setup.outputs.publish != 'true' uses: actions/download-artifact@v8 with: - pattern: rust-volume-* + pattern: rust-bins-* merge-multiple: true path: ./rust-bins - name: Place Rust binaries in Docker context for local scan @@ -304,7 +321,16 @@ jobs: echo "Placed pre-built Rust binary for ${arch}" fi done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ - name: Create BuildKit config for local scan build if: needs.setup.outputs.publish != 'true' run: | diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index f8d9cd21d..f8177061b 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -42,9 +42,10 @@ concurrency: jobs: - # ── Pre-build Rust volume server binaries natively ────────────────── - # Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour - # emulated cargo build into ~15 minutes of native compilation. + # ── Pre-build the Rust binaries natively ──────────────────────────── + # The volume server and the Rust maintenance worker, cross-compiled for + # amd64 and arm64 without QEMU, turning a 5-hour emulated cargo build into + # ~15 minutes of native compilation. build-rust-binaries: runs-on: ubuntu-22.04 strategy: @@ -83,7 +84,8 @@ jobs: ~/.cargo/registry ~/.cargo/git seaweed-volume/target - key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + seaweed-worker/target/${{ matrix.target }}/release + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-worker/Cargo.lock') }} restore-keys: | rust-docker-${{ matrix.target }}- @@ -103,13 +105,20 @@ jobs: cargo build --release --target ${{ matrix.target }} --no-default-features cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }} + - name: Build the Rust maintenance worker + run: | + cd seaweed-worker + cargo build --release -p weed-lance-worker --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-worker ../weed-worker-${{ matrix.arch }} + - name: Upload artifacts uses: actions/upload-artifact@v7 with: - name: rust-volume-${{ matrix.arch }} + name: rust-bins-${{ matrix.arch }} path: | weed-volume-large-disk-${{ matrix.arch }} weed-volume-normal-${{ matrix.arch }} + weed-worker-${{ matrix.arch }} # One job per (variant, platform) on a native runner, pushed by digest; # the merge job stitches the digests into one multi-arch tag. @@ -155,7 +164,7 @@ jobs: if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant uses: actions/download-artifact@v8 with: - pattern: rust-volume-* + pattern: rust-bins-* merge-multiple: true path: ./rust-bins @@ -170,7 +179,16 @@ jobs: echo "Placed pre-built Rust binary for ${arch}" fi done + mkdir -p docker/weed-worker-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-worker-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-worker-prebuilt/weed-worker-${arch}" + echo "Placed pre-built Rust worker for ${arch}" + fi + done ls -la docker/weed-volume-prebuilt/ + ls -la docker/weed-worker-prebuilt/ - name: Free Disk Space if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant