mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-31 17:30:42 +01:00
* test: add s3 integration tests for SQLite Cloud Backed Storage (#12235) SQLite CBS stores a database as named objects in a bucket and relies on conditional PUT — put-if-absent for new blocks and ETag compare-and-swap on the manifest — so concurrent clients cannot corrupt the database. The suite runs a weed mini cluster (conditions are evaluated under the object's write lock on the owner filer via the routed-write path) and covers: - the CBS storage primitives: create/put/fetch/list/delete/destroy - If-None-Match: * put-if-absent, success and 412 rejection - If-Match ETag compare-and-swap, including stale-ETag rejection and the re-read/retry loop CBS clients run on 412, on plain and versioned buckets - racing put-if-absent and manifest CAS writers: exactly one commit wins, every update survives - conditional CompleteMultipartUpload, sequential and raced - an end-to-end bcv-style upload, download-and-verify, and checkpoint A matching job is added to the S3 mutation regression workflow. * test/s3: drop issue references from the sqlite cbs suite comments * test/s3: abort every losing upload after the conditional multipart race * test/s3: assert the winning writer's body after the conditional write races * test/s3: spell out the get-returned etag in the cbs suite * test/s3: assert the completed multipart ETag the cbs suite feeds back The conditional-completion checks pass completeOut.ETag into If-Match; if the response carried no ETag the check would silently run unconditional. * test/s3: drop the unused generation parameter from the cbs block body
191 lines
6.0 KiB
YAML
191 lines
6.0 KiB
YAML
name: "S3 Mutation Regression Tests"
|
|
|
|
on:
|
|
pull_request:
|
|
paths:
|
|
- 'weed/s3api/**'
|
|
- 'test/s3/delete/**'
|
|
- 'test/s3/distributed_lock/**'
|
|
- 'test/s3/sqlite_cbs/**'
|
|
- 'test/s3/versioning/**'
|
|
- 'test/volume_server/framework/**'
|
|
- 'docker/compose/s3.json'
|
|
- '.github/workflows/s3-mutation-regression-tests.yml'
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }}
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }}
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
s3-versioning-regressions:
|
|
name: S3 Versioning Regression Tests
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 25
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v7
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
cache: ${{ runner.environment == 'github-hosted' }}
|
|
|
|
- name: Run S3 versioning regression tests
|
|
timeout-minutes: 20
|
|
working-directory: test/s3/versioning
|
|
run: |
|
|
set -x
|
|
# Run every versioning test, so a regression test lands covered instead
|
|
# of waiting for someone to remember this file. Name a test in EXCLUDE,
|
|
# with the reason, to keep it out.
|
|
#
|
|
# TestVersioningPagination*: opt-in stress tests that build 1500+
|
|
# versions. They self-skip without ENABLE_STRESS_TESTS and have their
|
|
# own make target, so this gate should not carry them.
|
|
EXCLUDE='TestVersioningPagination.*'
|
|
tests=$(go test . -list '.*' | grep '^Test' | sort -u)
|
|
# An empty list would make -run match nothing and pass this job vacuously.
|
|
[ -n "$tests" ] || { echo "listed no versioning tests"; exit 1; }
|
|
selected=$(echo "$tests" | grep -vE "^($EXCLUDE)$")
|
|
[ -n "$selected" ] || { echo "EXCLUDE matched every test"; exit 1; }
|
|
echo "running $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) versioning tests"
|
|
# make swallows a lone trailing $, taking the anchor with it, so escape it.
|
|
make test-with-server TEST_PATTERN="^($(echo "$selected" | paste -sd'|' -))"'$$'
|
|
|
|
- name: Show server logs on failure
|
|
if: failure()
|
|
working-directory: test/s3/versioning
|
|
run: |
|
|
echo "=== Server Logs ==="
|
|
if [ -f weed-test.log ]; then
|
|
tail -100 weed-test.log
|
|
fi
|
|
|
|
- name: Upload versioning logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: s3-versioning-regression-logs
|
|
path: test/s3/versioning/weed-test*.log
|
|
retention-days: 3
|
|
|
|
s3-delete-regressions:
|
|
name: S3 Delete Regression Tests
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 20
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v7
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
cache: ${{ runner.environment == 'github-hosted' }}
|
|
|
|
- name: Run S3 delete regression tests
|
|
timeout-minutes: 15
|
|
working-directory: test/s3/delete
|
|
run: |
|
|
set -x
|
|
make test-with-server
|
|
|
|
- name: Show server logs on failure
|
|
if: failure()
|
|
working-directory: test/s3/delete
|
|
run: |
|
|
echo "=== Server Logs ==="
|
|
if [ -f weed-test.log ]; then
|
|
tail -100 weed-test.log
|
|
fi
|
|
|
|
- name: Upload delete logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: s3-delete-regression-logs
|
|
path: test/s3/delete/weed-test*.log
|
|
retention-days: 3
|
|
|
|
s3-distributed-lock-regressions:
|
|
name: S3 Distributed Lock Regression Tests
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v7
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
cache: ${{ runner.environment == 'github-hosted' }}
|
|
|
|
- name: Build SeaweedFS
|
|
run: |
|
|
go build -o weed/weed -buildvcs=false ./weed
|
|
|
|
- name: Run distributed lock regressions
|
|
timeout-minutes: 25
|
|
env:
|
|
TMPDIR: ${{ github.workspace }}/test/s3/distributed_lock/tmp
|
|
S3_DISTRIBUTED_LOCK_KEEP_LOGS: "1"
|
|
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
|
run: |
|
|
set -x
|
|
mkdir -p "$TMPDIR"
|
|
go test -v -count=1 -timeout=20m ./test/s3/distributed_lock
|
|
|
|
- name: Upload distributed lock logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: s3-distributed-lock-regression-logs
|
|
path: test/s3/distributed_lock/tmp/seaweedfs_s3_distributed_lock_*
|
|
retention-days: 3
|
|
|
|
s3-sqlite-cbs-regressions:
|
|
name: S3 SQLite CBS Regression Tests
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 20
|
|
|
|
steps:
|
|
- name: Check out code
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Set up Go
|
|
uses: actions/setup-go@v7
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
cache: ${{ runner.environment == 'github-hosted' }}
|
|
|
|
- name: Build SeaweedFS
|
|
run: |
|
|
go build -o weed/weed -buildvcs=false ./weed
|
|
|
|
- name: Run sqlite cbs regressions
|
|
timeout-minutes: 15
|
|
env:
|
|
TMPDIR: ${{ github.workspace }}/test/s3/sqlite_cbs/tmp
|
|
S3_SQLITE_CBS_KEEP_LOGS: "1"
|
|
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
|
run: |
|
|
set -x
|
|
mkdir -p "$TMPDIR"
|
|
go test -v -count=1 -timeout=12m ./test/s3/sqlite_cbs
|
|
|
|
- name: Upload sqlite cbs logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: s3-sqlite-cbs-regression-logs
|
|
path: test/s3/sqlite_cbs/tmp/seaweedfs_sqlite_cbs_*
|
|
retention-days: 3
|