mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-26 23:10:44 +01:00
Chris Lu
52d91008ac
GitHub is retiring ubuntu-22.04; move all 94 runs-on pins across 40 workflows to ubuntu-24.04. rust_binaries_release: the arm64 cross step was jammy-era — noble keeps apt sources in deb822 ubuntu.sources (the sources.list edit was a silent no-op that left amd64 repos open to arm64 and 404'd) and ports suites must say noble. Pin deb822 stanzas to amd64 and add noble ports entries.
352 lines
13 KiB
YAML
352 lines
13 KiB
YAML
name: "Rust Volume Server Tests"
|
|
|
|
on:
|
|
pull_request:
|
|
branches: [ master ]
|
|
paths:
|
|
- 'seaweed-volume/**'
|
|
- 'seaweed-common/**'
|
|
- 'test/volume_server/**'
|
|
- 'weed/pb/volume_server.proto'
|
|
- 'weed/pb/volume_server_pb/**'
|
|
- '.github/workflows/rust-volume-server-tests.yml'
|
|
push:
|
|
branches: [ master, main ]
|
|
paths:
|
|
- 'seaweed-volume/**'
|
|
- 'seaweed-common/**'
|
|
- 'test/volume_server/**'
|
|
- 'weed/pb/volume_server.proto'
|
|
- 'weed/pb/volume_server_pb/**'
|
|
- '.github/workflows/rust-volume-server-tests.yml'
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
|
|
jobs:
|
|
changes:
|
|
name: Detect changed paths
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
rust: ${{ steps.filter.outputs.rust }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v7
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Filter changed paths
|
|
id: filter
|
|
uses: dorny/paths-filter@v4
|
|
with:
|
|
filters: |
|
|
rust:
|
|
- 'seaweed-volume/**'
|
|
- '.github/workflows/rust-volume-server-tests.yml'
|
|
|
|
build-test-binaries:
|
|
name: Build Test Binaries
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v7
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
cache: ${{ runner.environment == 'github-hosted' }}
|
|
|
|
- name: Install Rust toolchain
|
|
uses: dtolnay/rust-toolchain@stable
|
|
|
|
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
|
# target/ can carry C objects built against a different glibc than we link against.
|
|
- name: Fingerprint build toolchain
|
|
id: toolchain
|
|
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Cache cargo registry and target
|
|
if: runner.environment == 'github-hosted'
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: |
|
|
~/.cargo/registry
|
|
~/.cargo/git
|
|
seaweed-volume/target
|
|
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
|
|
restore-keys: |
|
|
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
|
|
|
- name: Build Go weed binary
|
|
run: |
|
|
cd weed
|
|
go build -tags 5BytesOffset -o weed .
|
|
chmod +x weed
|
|
./weed version
|
|
|
|
- name: Build Rust volume binary
|
|
run: cd seaweed-volume && cargo build --release
|
|
|
|
- name: Package binaries
|
|
run: tar cf rust-test-binaries.tar weed/weed seaweed-volume/target/release/weed-volume
|
|
|
|
- name: Upload test binaries
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
# archive layout mirrors the workspace: weed/weed and
|
|
# seaweed-volume/target/release/weed-volume
|
|
name: rust-test-binaries
|
|
if-no-files-found: error
|
|
retention-days: 1
|
|
path: rust-test-binaries.tar
|
|
|
|
rust-unit-tests:
|
|
name: Rust Unit Tests
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 15
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Install Rust toolchain
|
|
uses: dtolnay/rust-toolchain@stable
|
|
|
|
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
|
# target/ can carry C objects built against a different glibc than we link against.
|
|
- name: Fingerprint build toolchain
|
|
id: toolchain
|
|
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Cache cargo registry and target
|
|
if: runner.environment == 'github-hosted'
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: |
|
|
~/.cargo/registry
|
|
~/.cargo/git
|
|
seaweed-volume/target
|
|
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }}
|
|
restore-keys: |
|
|
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
|
|
|
- name: Build Rust volume server
|
|
run: cd seaweed-volume && cargo build --release
|
|
|
|
# The crate is warning-free under clippy as of the sweep that added
|
|
# this step. Uncomment to make that a gate; `[lints.clippy]` in
|
|
# seaweed-volume/Cargo.toml is where crate-wide exceptions live.
|
|
# - name: Clippy
|
|
# run: cd seaweed-volume && cargo clippy --all-targets -- -D warnings
|
|
|
|
# The crate is rustfmt-clean as of the PR that added this step.
|
|
# Uncomment to keep it that way.
|
|
# - name: Check formatting
|
|
# run: cd seaweed-volume && cargo fmt --check
|
|
|
|
# seaweed-common is a path dependency of this crate, not a member of its
|
|
# workspace, so the run below does not reach its own tests. It builds into
|
|
# this job's cached target directory, and the cache key above covers the
|
|
# shared crate's lock, so the aws-lc-sys that rustls pulls in is restored
|
|
# with the cache instead of compiled from scratch on every run.
|
|
- name: Run shared-crate unit tests
|
|
env:
|
|
CARGO_TARGET_DIR: ${{ github.workspace }}/seaweed-volume/target
|
|
run: cd seaweed-common && cargo test
|
|
|
|
- name: Run Rust unit tests
|
|
run: cd seaweed-volume && cargo test
|
|
|
|
- name: Run Rust unit tests (redb experimental cursor)
|
|
run: cd seaweed-volume && cargo test --features redb-experimental-cursor --lib storage::needle_map
|
|
|
|
rust-unit-tests-windows:
|
|
name: Rust Unit Tests (Windows)
|
|
runs-on: windows-latest
|
|
timeout-minutes: 30
|
|
needs: [changes]
|
|
if: needs.changes.outputs.rust == 'true'
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Install Rust toolchain
|
|
uses: dtolnay/rust-toolchain@stable
|
|
|
|
# No glibc on Windows: key the cache on the toolchain and OS only.
|
|
- name: Fingerprint build toolchain
|
|
id: toolchain
|
|
run: echo "fingerprint=windows-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Cache cargo registry and target
|
|
if: runner.environment == 'github-hosted'
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: |
|
|
~/.cargo/registry
|
|
~/.cargo/git
|
|
seaweed-volume/target
|
|
key: rust-windows-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
|
restore-keys: |
|
|
rust-windows-${{ steps.toolchain.outputs.fingerprint }}-
|
|
|
|
- name: Run Rust unit tests
|
|
run: cd seaweed-volume && cargo test
|
|
|
|
- name: Run Rust unit tests (redb experimental cursor)
|
|
run: cd seaweed-volume && cargo test --features redb-experimental-cursor --lib storage::needle_map
|
|
|
|
rust-integration-tests:
|
|
name: Rust Integration Tests
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 30
|
|
needs: [build-test-binaries]
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v7
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
cache: ${{ runner.environment == 'github-hosted' }}
|
|
|
|
- name: Download test binaries
|
|
uses: actions/download-artifact@v7
|
|
with:
|
|
name: rust-test-binaries
|
|
|
|
- name: Restore binaries
|
|
run: |
|
|
tar xf rust-test-binaries.tar
|
|
chmod +x weed/weed seaweed-volume/target/release/weed-volume
|
|
./weed/weed version
|
|
|
|
- name: Run integration tests
|
|
env:
|
|
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
|
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
|
|
run: |
|
|
echo "Running Rust volume server integration tests..."
|
|
go test -v -count=1 -timeout=15m ./test/volume_server/rust/...
|
|
|
|
- name: Collect logs on failure
|
|
if: failure()
|
|
run: |
|
|
mkdir -p /tmp/rust-volume-server-it-logs
|
|
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-server-it-logs/ \; || true
|
|
|
|
- name: Archive logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: rust-volume-server-integration-test-logs
|
|
path: /tmp/rust-volume-server-it-logs/
|
|
if-no-files-found: warn
|
|
retention-days: 7
|
|
|
|
- name: Test summary
|
|
if: always()
|
|
run: |
|
|
echo "## Rust Volume Server Integration Test Summary" >> "$GITHUB_STEP_SUMMARY"
|
|
echo "- Suite: test/volume_server/rust" >> "$GITHUB_STEP_SUMMARY"
|
|
echo "- Command: go test -v -count=1 -timeout=15m ./test/volume_server/rust/..." >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
rust-volume-go-tests:
|
|
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 45
|
|
needs: [build-test-binaries]
|
|
env:
|
|
# Keep in step with the length of matrix.shard below.
|
|
SHARD_COUNT: 3
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
test-type: [grpc, http]
|
|
shard: [1, 2, 3]
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v7
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
cache: ${{ runner.environment == 'github-hosted' }}
|
|
|
|
- name: Download test binaries
|
|
uses: actions/download-artifact@v7
|
|
with:
|
|
name: rust-test-binaries
|
|
|
|
- name: Restore binaries
|
|
run: |
|
|
tar xf rust-test-binaries.tar
|
|
chmod +x weed/weed seaweed-volume/target/release/weed-volume
|
|
./weed/weed version
|
|
./seaweed-volume/target/release/weed-volume --help >/dev/null || true
|
|
|
|
# Dealing the listed tests out one by one keeps the shards even. Bucketing
|
|
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
|
|
# the 114 grpc tests and ran nearly twice as long as the other two shards.
|
|
- name: Select this shard's tests
|
|
env:
|
|
TEST_TYPE: ${{ matrix.test-type }}
|
|
SHARD: ${{ matrix.shard }}
|
|
run: |
|
|
tests=$(go test -tags 5BytesOffset ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u)
|
|
# An empty list would make -run match nothing and the shard pass vacuously.
|
|
[ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; }
|
|
selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1')
|
|
echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests"
|
|
echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV"
|
|
|
|
- name: Run volume server integration tests with Rust volume
|
|
env:
|
|
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
|
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
|
|
VOLUME_SERVER_IMPL: rust
|
|
run: |
|
|
echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..."
|
|
go test -v -count=1 -tags 5BytesOffset -timeout=30m ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
|
|
|
|
- name: Collect logs on failure
|
|
if: failure()
|
|
run: |
|
|
mkdir -p /tmp/rust-volume-go-test-logs
|
|
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-go-test-logs/ \; || true
|
|
|
|
- name: Archive logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: rust-volume-go-test-logs-${{ matrix.test-type }}-shard${{ matrix.shard }}
|
|
path: /tmp/rust-volume-go-test-logs/
|
|
if-no-files-found: warn
|
|
retention-days: 7
|
|
|
|
- name: Test summary
|
|
if: always()
|
|
run: |
|
|
echo "## Rust Volume - Go Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
|
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
|
|
echo "- Volume server: Rust (VOLUME_SERVER_IMPL=rust)" >> "$GITHUB_STEP_SUMMARY"
|