mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-08 15:41:15 +02:00
Chris Lu
5a515adab2
* s3: HEAD with partNumber reports the part's size and range HeadObject set its headers from the total object size and then only validated the partNumber, so a client probing part 1 with HEAD got the whole object's Content-Length and a 200 while the same GET returned the part's size, a Content-Range and a 206. Resolve the part's byte range before the headers are written, through the range logic GetObject already used, and answer a partNumber HEAD as the ranged HEAD that AWS documents. * s3: answer an unsatisfiable partNumber with 416 InvalidPartNumber GET and HEAD rejected a partNumber past the number of parts with 400 InvalidPart, the code for a missing part in CompleteMultipartUpload. AWS answers a read of a part that does not exist with 416 InvalidPartNumber, which lets a client probing for the part count tell the two apart. The ceph suite pins RGW's 400 InvalidPart here, so the s3tests jobs patch that expectation the way they already patch prefix ordering. * s3: keep the whole-object checksum off a partNumber response The stored checksum covers the whole object, so it is already withheld from a ranged read. A partNumber HEAD now describes one part while the request carries no Range header, so exclude it there too rather than handing a client a checksum that does not match the bytes described. * s3: resolve a partNumber against the parts the object records Completion accepts ascending, not consecutive, part numbers, so the part count is not the highest part number. Comparing the two rejected an uploaded part 3 of a two-part object, and let a request for the absent part 2 fall through to the positional chunk lookup and serve part 3's bytes. Ask the recorded boundaries for the part instead, and keep the count comparison for objects written before boundaries were stored. * s3: apply a client Range within the part on HEAD too GET narrowed the part by a Range sent alongside partNumber; HEAD reported the whole part, so the two disagreed again for a request that carries both. Move the narrowing into the shared range lookup so either verb describes the same bytes.
1242 lines
62 KiB
YAML
1242 lines
62 KiB
YAML
name: "Ceph S3 tests"
|
|
|
|
on:
|
|
push:
|
|
branches: [ master ]
|
|
paths:
|
|
- 'weed/s3api/**'
|
|
- 'weed/filer/**'
|
|
- 'weed/server/**'
|
|
- 'weed/iam/**'
|
|
- 'test/s3/compatibility/**'
|
|
- 'go.mod'
|
|
- 'go.sum'
|
|
- '.github/workflows/s3tests.yml'
|
|
pull_request:
|
|
branches: [ master ]
|
|
paths:
|
|
- 'weed/s3api/**'
|
|
- 'weed/filer/**'
|
|
- 'weed/server/**'
|
|
- 'weed/iam/**'
|
|
- 'test/s3/compatibility/**'
|
|
- 'go.mod'
|
|
- 'go.sum'
|
|
- '.github/workflows/s3tests.yml'
|
|
|
|
concurrency:
|
|
group: ${{ github.head_ref }}/s3tests
|
|
cancel-in-progress: true
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
basic-s3-tests:
|
|
name: Basic S3 tests (KV store)
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 15
|
|
steps:
|
|
- name: Check out code into the Go module directory
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Set up Go 1.x
|
|
uses: actions/setup-go@v7
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v7
|
|
with:
|
|
python-version: '3.9'
|
|
|
|
- name: Clone s3-tests
|
|
run: |
|
|
git clone https://github.com/ceph/s3-tests.git
|
|
cd s3-tests
|
|
sudo apt-get update -qq
|
|
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
|
pip install -r requirements.txt
|
|
pip install tox
|
|
pip install -e .
|
|
|
|
- name: Fix S3 tests bucket creation conflicts
|
|
run: |
|
|
python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py
|
|
env:
|
|
S3_TESTS_PATH: s3-tests
|
|
|
|
- name: Run Basic S3 tests
|
|
timeout-minutes: 15
|
|
env:
|
|
S3TEST_CONF: ../docker/compose/s3tests.conf
|
|
shell: bash
|
|
run: |
|
|
cd weed
|
|
go install -tags s3tests -buildvcs=false
|
|
set -x
|
|
# Create clean data directory for this test run
|
|
export WEED_DATA_DIR="/tmp/seaweedfs-s3tests-$(date +%s)"
|
|
mkdir -p "$WEED_DATA_DIR"
|
|
weed -v 3 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
|
|
-dir="$WEED_DATA_DIR" \
|
|
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
|
|
-volume.max=100 -volume.preStopSeconds=1 \
|
|
-master.port=9333 -volume.port=8080 -filer.port=8888 -s3.port=8000 -metricsPort=9324 \
|
|
-s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
|
|
pid=$!
|
|
|
|
# Wait for all SeaweedFS components to be ready
|
|
echo "Waiting for SeaweedFS components to start..."
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:9333/cluster/status > /dev/null 2>&1; then
|
|
echo "Master server is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for master server... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:8080/status > /dev/null 2>&1; then
|
|
echo "Volume server is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for volume server... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:8888/ > /dev/null 2>&1; then
|
|
echo "Filer is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for filer... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:8000/ > /dev/null 2>&1; then
|
|
echo "S3 server is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for S3 server... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
echo "All SeaweedFS components are ready!"
|
|
cd ../s3-tests
|
|
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests/functional/test_s3.py
|
|
# The suite expects RGW's 400 InvalidPart for a partNumber past the last
|
|
# part; AWS answers 416 InvalidPartNumber, which is what we return.
|
|
sed -i "/# request PartNumber out of range/,+4{s/assert status == 400/assert status == 416/; s/assert error_code == 'InvalidPart'/assert error_code == 'InvalidPartNumber'/}" s3tests/functional/test_s3.py
|
|
|
|
# Debug: Show the config file contents
|
|
echo "=== S3 Config File Contents ==="
|
|
cat ../docker/compose/s3tests.conf
|
|
echo "=== End Config ==="
|
|
|
|
# Additional wait for S3-Filer integration to be fully ready
|
|
echo "Waiting additional 10 seconds for S3-Filer integration..."
|
|
sleep 10
|
|
|
|
# Test S3 connection before running tests
|
|
echo "Testing S3 connection..."
|
|
for i in {1..10}; do
|
|
if curl -s -f http://localhost:8000/ > /dev/null 2>&1; then
|
|
echo "S3 connection test successful"
|
|
break
|
|
fi
|
|
echo "S3 connection test failed, retrying... ($i/10)"
|
|
sleep 2
|
|
done
|
|
|
|
echo "✅ S3 server is responding, starting tests..."
|
|
|
|
# Spawn the lifecycle worker so test_lifecycle_expiration etc. have
|
|
# something driving deletions. The s3tests build tag rescales one
|
|
# day to LifeCycleInterval=10s, so a 1d rule fires within ~10s of
|
|
# the upload's mtime; -dispatch / -checkpoint defaults are already
|
|
# tightened under the same build tag.
|
|
LC_LOG=/tmp/lifecycle-worker.log
|
|
# -debug routes glog to stderr so the bootstrap walker's progress
|
|
# shows up in $LC_LOG; without it weed shell silences glog.
|
|
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18000 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
|
|
| weed shell -debug -master=localhost:9333 \
|
|
> "$LC_LOG" 2>&1 &
|
|
lc_pid=$!
|
|
# Aliveness check: a bad shell command exits in <1s and the suite
|
|
# would otherwise just timeout the expiration tests with no signal.
|
|
sleep 2
|
|
if ! kill -0 "$lc_pid" 2>/dev/null; then
|
|
echo "lifecycle worker died on startup"
|
|
tail -50 "$LC_LOG" 2>/dev/null || true
|
|
exit 1
|
|
fi
|
|
echo "lifecycle worker pid=$lc_pid"
|
|
|
|
# bash -e exits the step on the first tox failure, so move teardown
|
|
# into a trap to guarantee the worker log + data dir reach the runner.
|
|
cleanup() {
|
|
status=$?
|
|
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
|
|
# bash fallback if it ignores TERM. Reading the log AFTER the
|
|
# graceful-stop window catches the bootstrap walker's progress.
|
|
kill -TERM "$lc_pid" 2>/dev/null || true
|
|
kill -TERM "$pid" 2>/dev/null || true
|
|
sleep 1
|
|
if [ "$status" -ne 0 ]; then
|
|
echo "=== lifecycle worker log (tail) ==="
|
|
tail -200 "$LC_LOG" 2>/dev/null || true
|
|
fi
|
|
kill -9 "$lc_pid" 2>/dev/null || true
|
|
kill -9 "$pid" 2>/dev/null || true
|
|
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
tox -- \
|
|
s3tests/functional/test_s3.py::test_bucket_list_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_list_distinct \
|
|
s3tests/functional/test_s3.py::test_bucket_list_many \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_many \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_encoding_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_list_encoding_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix_ends_with_delimiter \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_alt \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_alt \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix_underscore \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_percentage \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_percentage \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_whitespace \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_whitespace \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_dot \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_dot \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_unreadable \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_none \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_none \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_not_skip_special \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_alt \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_alt \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_delimiter_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_defaultempty \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_alt \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_alt \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_none \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_none \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_unreadable \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_unreadable \
|
|
s3tests/functional/test_s3.py::test_bucket_list_maxkeys_one \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_one \
|
|
s3tests/functional/test_s3.py::test_bucket_list_maxkeys_zero \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \
|
|
s3tests/functional/test_s3.py::test_bucket_list_maxkeys_none \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_none \
|
|
s3tests/functional/test_s3.py::test_bucket_list_unordered \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_unordered \
|
|
s3tests/functional/test_s3.py::test_bucket_list_maxkeys_invalid \
|
|
s3tests/functional/test_s3.py::test_bucket_list_marker_none \
|
|
s3tests/functional/test_s3.py::test_bucket_list_marker_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_continuationtoken_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_continuationtoken \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \
|
|
s3tests/functional/test_s3.py::test_bucket_list_marker_unreadable \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_startafter_unreadable \
|
|
s3tests/functional/test_s3.py::test_bucket_list_marker_not_in_list \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_startafter_not_in_list \
|
|
s3tests/functional/test_s3.py::test_bucket_list_marker_after_list \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_startafter_after_list \
|
|
s3tests/functional/test_s3.py::test_bucket_list_return_data \
|
|
s3tests/functional/test_s3.py::test_bucket_list_objects_anonymous \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_objects_anonymous \
|
|
s3tests/functional/test_s3.py::test_bucket_list_objects_anonymous_fail \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_objects_anonymous_fail \
|
|
s3tests/functional/test_s3.py::test_bucket_list_long_name \
|
|
s3tests/functional/test_s3.py::test_bucket_list_special_prefix \
|
|
s3tests/functional/test_s3.py::test_bucket_delete_notexist \
|
|
s3tests/functional/test_s3.py::test_bucket_create_delete \
|
|
s3tests/functional/test_s3.py::test_object_read_not_exist \
|
|
s3tests/functional/test_s3.py::test_multi_object_delete \
|
|
s3tests/functional/test_s3.py::test_multi_objectv2_delete \
|
|
s3tests/functional/test_s3.py::test_object_head_zero_bytes \
|
|
s3tests/functional/test_s3.py::test_object_write_check_etag \
|
|
s3tests/functional/test_s3.py::test_object_write_cache_control \
|
|
s3tests/functional/test_s3.py::test_object_write_expires \
|
|
s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
|
|
s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
|
|
s3tests/functional/test_s3.py::test_object_write_file \
|
|
s3tests/functional/test_s3.py::test_post_object_invalid_date_format \
|
|
s3tests/functional/test_s3.py::test_post_object_no_key_specified \
|
|
s3tests/functional/test_s3.py::test_post_object_missing_signature \
|
|
s3tests/functional/test_s3.py::test_post_object_condition_is_case_sensitive \
|
|
s3tests/functional/test_s3.py::test_post_object_expires_is_case_sensitive \
|
|
s3tests/functional/test_s3.py::test_post_object_missing_expires_condition \
|
|
s3tests/functional/test_s3.py::test_post_object_missing_conditions_list \
|
|
s3tests/functional/test_s3.py::test_post_object_upload_size_limit_exceeded \
|
|
s3tests/functional/test_s3.py::test_post_object_missing_content_length_argument \
|
|
s3tests/functional/test_s3.py::test_post_object_invalid_content_length_argument \
|
|
s3tests/functional/test_s3.py::test_post_object_upload_size_below_minimum \
|
|
s3tests/functional/test_s3.py::test_post_object_empty_conditions \
|
|
s3tests/functional/test_s3.py::test_get_object_ifmatch_good \
|
|
s3tests/functional/test_s3.py::test_get_object_ifnonematch_good \
|
|
s3tests/functional/test_s3.py::test_get_object_ifmatch_failed \
|
|
s3tests/functional/test_s3.py::test_get_object_ifnonematch_failed \
|
|
s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
|
|
s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
|
|
s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
|
|
s3tests/functional/test_s3.py::test_bucket_head \
|
|
s3tests/functional/test_s3.py::test_bucket_head_notexist \
|
|
s3tests/functional/test_s3.py::test_object_raw_authenticated \
|
|
s3tests/functional/test_s3.py::test_object_raw_authenticated_bucket_acl \
|
|
s3tests/functional/test_s3.py::test_object_raw_authenticated_object_acl \
|
|
s3tests/functional/test_s3.py::test_object_raw_authenticated_object_gone \
|
|
s3tests/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_range_zero \
|
|
s3tests/functional/test_s3.py::test_object_anon_put \
|
|
s3tests/functional/test_s3.py::test_object_put_authenticated \
|
|
s3tests/functional/test_s3.py::test_bucket_recreate_overwrite_acl \
|
|
s3tests/functional/test_s3.py::test_bucket_recreate_new_acl \
|
|
s3tests/functional/test_s3.py::test_buckets_create_then_list \
|
|
s3tests/functional/test_s3.py::test_buckets_list_ctime \
|
|
s3tests/functional/test_s3.py::test_list_buckets_invalid_auth \
|
|
s3tests/functional/test_s3.py::test_list_buckets_bad_auth \
|
|
s3tests/functional/test_s3.py::test_bucket_create_naming_good_contains_period \
|
|
s3tests/functional/test_s3.py::test_bucket_create_naming_good_contains_hyphen \
|
|
s3tests/functional/test_s3.py::test_bucket_list_special_prefix \
|
|
s3tests/functional/test_s3.py::test_object_copy_zero_size \
|
|
s3tests/functional/test_s3.py::test_object_copy_same_bucket \
|
|
s3tests/functional/test_s3.py::test_object_copy_to_itself \
|
|
s3tests/functional/test_s3.py::test_object_copy_diff_bucket \
|
|
s3tests/functional/test_s3.py::test_object_copy_canned_acl \
|
|
s3tests/functional/test_s3.py::test_object_copy_bucket_not_found \
|
|
s3tests/functional/test_s3.py::test_object_copy_key_not_found \
|
|
s3tests/functional/test_s3.py::test_multipart_copy_small \
|
|
s3tests/functional/test_s3.py::test_multipart_copy_without_range \
|
|
s3tests/functional/test_s3.py::test_multipart_copy_special_names \
|
|
s3tests/functional/test_s3.py::test_multipart_copy_multiple_sizes \
|
|
s3tests/functional/test_s3.py::test_multipart_get_part \
|
|
s3tests/functional/test_s3.py::test_multipart_upload \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_empty \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_multiple_sizes \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_contents \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_size_too_small \
|
|
s3tests/functional/test_s3.py::test_multipart_resend_first_finishes_last \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_resend_part \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_missing_part \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_incorrect_etag \
|
|
s3tests/functional/test_s3.py::test_abort_multipart_upload \
|
|
s3tests/functional/test_s3.py::test_list_multipart_upload \
|
|
s3tests/functional/test_s3.py::test_atomic_read_1mb \
|
|
s3tests/functional/test_s3.py::test_atomic_read_4mb \
|
|
s3tests/functional/test_s3.py::test_atomic_read_8mb \
|
|
s3tests/functional/test_s3.py::test_atomic_write_1mb \
|
|
s3tests/functional/test_s3.py::test_atomic_write_4mb \
|
|
s3tests/functional/test_s3.py::test_atomic_write_8mb \
|
|
s3tests/functional/test_s3.py::test_atomic_dual_write_1mb \
|
|
s3tests/functional/test_s3.py::test_atomic_dual_write_4mb \
|
|
s3tests/functional/test_s3.py::test_atomic_dual_write_8mb \
|
|
s3tests/functional/test_s3.py::test_atomic_multipart_upload_write \
|
|
s3tests/functional/test_s3.py::test_ranged_request_response_code \
|
|
s3tests/functional/test_s3.py::test_ranged_big_request_response_code \
|
|
s3tests/functional/test_s3.py::test_ranged_request_skip_leading_bytes_response_code \
|
|
s3tests/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \
|
|
s3tests/functional/test_s3.py::test_copy_object_ifmatch_good \
|
|
s3tests/functional/test_s3.py::test_copy_object_ifnonematch_failed \
|
|
s3tests/functional/test_s3.py::test_copy_object_ifmatch_failed \
|
|
s3tests/functional/test_s3.py::test_copy_object_ifnonematch_good \
|
|
s3tests/functional/test_s3.py::test_lifecycle_set \
|
|
s3tests/functional/test_s3.py::test_lifecycle_get \
|
|
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
|
|
s3tests/functional/test_s3.py::test_lifecycle_expiration \
|
|
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
|
|
# cleanup() trap handles worker/server kill + data dir wipe.
|
|
|
|
versioning-tests:
|
|
name: S3 Versioning & Object Lock tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 15
|
|
steps:
|
|
- name: Check out code into the Go module directory
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Set up Go 1.x
|
|
uses: actions/setup-go@v7
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v7
|
|
with:
|
|
python-version: '3.9'
|
|
|
|
- name: Clone s3-tests
|
|
run: |
|
|
git clone https://github.com/ceph/s3-tests.git
|
|
cd s3-tests
|
|
sudo apt-get update -qq
|
|
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
|
pip install -r requirements.txt
|
|
pip install tox
|
|
pip install -e .
|
|
|
|
- name: Fix S3 tests bucket creation conflicts
|
|
run: |
|
|
python3 test/s3/compatibility/fix_s3_tests_bucket_conflicts.py
|
|
env:
|
|
S3_TESTS_PATH: s3-tests
|
|
|
|
- name: Run S3 Object Lock, Retention, and Versioning tests
|
|
timeout-minutes: 15
|
|
shell: bash
|
|
run: |
|
|
cd weed
|
|
go install -buildvcs=false
|
|
set -x
|
|
# Create clean data directory for this test run
|
|
export WEED_DATA_DIR="/tmp/seaweedfs-objectlock-versioning-$(date +%s)"
|
|
mkdir -p "$WEED_DATA_DIR"
|
|
|
|
# Verify S3 config file exists
|
|
echo "Checking S3 config file: $GITHUB_WORKSPACE/docker/compose/s3.json"
|
|
ls -la "$GITHUB_WORKSPACE/docker/compose/s3.json"
|
|
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
|
|
-dir="$WEED_DATA_DIR" \
|
|
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
|
|
-volume.max=100 -volume.preStopSeconds=1 \
|
|
-master.port=9334 -volume.port=8081 -filer.port=8889 -s3.port=8001 -metricsPort=9325 \
|
|
-s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
|
|
pid=$!
|
|
|
|
# Wait for all SeaweedFS components to be ready
|
|
echo "Waiting for SeaweedFS components to start..."
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:9334/cluster/status > /dev/null 2>&1; then
|
|
echo "Master server is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for master server... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:8081/status > /dev/null 2>&1; then
|
|
echo "Volume server is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for volume server... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:8889/ > /dev/null 2>&1; then
|
|
echo "Filer is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for filer... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:8001/ > /dev/null 2>&1; then
|
|
echo "S3 server is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for S3 server... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
echo "All SeaweedFS components are ready!"
|
|
cd ../s3-tests
|
|
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests/functional/test_s3.py
|
|
# Create and update s3tests.conf to use port 8001
|
|
cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-versioning.conf
|
|
sed -i 's/port = 8000/port = 8001/g' ../docker/compose/s3tests-versioning.conf
|
|
sed -i 's/:8000/:8001/g' ../docker/compose/s3tests-versioning.conf
|
|
sed -i 's/localhost:8000/localhost:8001/g' ../docker/compose/s3tests-versioning.conf
|
|
sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8001/g' ../docker/compose/s3tests-versioning.conf
|
|
# Use the configured bucket prefix from config and do not override with unique prefixes
|
|
# This avoids mismatch in tests that rely on a fixed provided name
|
|
export S3TEST_CONF=../docker/compose/s3tests-versioning.conf
|
|
|
|
# Debug: Show the config file contents
|
|
echo "=== S3 Config File Contents ==="
|
|
cat ../docker/compose/s3tests-versioning.conf
|
|
echo "=== End Config ==="
|
|
|
|
# Additional wait for S3-Filer integration to be fully ready
|
|
echo "Waiting additional 10 seconds for S3-Filer integration..."
|
|
sleep 10
|
|
|
|
# Test S3 connection before running tests
|
|
echo "Testing S3 connection..."
|
|
for i in {1..10}; do
|
|
if curl -s -f http://localhost:8001/ > /dev/null 2>&1; then
|
|
echo "S3 connection test successful"
|
|
break
|
|
fi
|
|
echo "S3 connection test failed, retrying... ($i/10)"
|
|
sleep 2
|
|
done
|
|
|
|
# Force cleanup any existing buckets to avoid conflicts
|
|
echo "Cleaning up any existing buckets..."
|
|
python3 -c "
|
|
import boto3
|
|
from botocore.exceptions import ClientError
|
|
try:
|
|
s3 = boto3.client('s3',
|
|
endpoint_url='http://localhost:8001',
|
|
aws_access_key_id='0555b35654ad1656d804',
|
|
aws_secret_access_key='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==')
|
|
buckets = s3.list_buckets()['Buckets']
|
|
for bucket in buckets:
|
|
bucket_name = bucket['Name']
|
|
print(f'Deleting bucket: {bucket_name}')
|
|
try:
|
|
# Delete all objects first
|
|
objects = s3.list_objects_v2(Bucket=bucket_name)
|
|
if 'Contents' in objects:
|
|
for obj in objects['Contents']:
|
|
s3.delete_object(Bucket=bucket_name, Key=obj['Key'])
|
|
# Delete all versions if versioning enabled
|
|
versions = s3.list_object_versions(Bucket=bucket_name)
|
|
if 'Versions' in versions:
|
|
for version in versions['Versions']:
|
|
s3.delete_object(Bucket=bucket_name, Key=version['Key'], VersionId=version['VersionId'])
|
|
if 'DeleteMarkers' in versions:
|
|
for marker in versions['DeleteMarkers']:
|
|
s3.delete_object(Bucket=bucket_name, Key=marker['Key'], VersionId=marker['VersionId'])
|
|
# Delete bucket
|
|
s3.delete_bucket(Bucket=bucket_name)
|
|
except ClientError as e:
|
|
print(f'Error deleting bucket {bucket_name}: {e}')
|
|
except Exception as e:
|
|
print(f'Cleanup failed: {e}')
|
|
" || echo "Cleanup completed with some errors (expected)"
|
|
|
|
# Run versioning and object lock tests once (avoid duplicates)
|
|
tox -- s3tests/functional/test_s3.py -k "object_lock or versioning" --tb=short
|
|
kill -9 $pid || true
|
|
# Clean up data directory
|
|
rm -rf "$WEED_DATA_DIR" || true
|
|
|
|
cors-tests:
|
|
name: S3 CORS tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Check out code into the Go module directory
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Set up Go 1.x
|
|
uses: actions/setup-go@v7
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v7
|
|
with:
|
|
python-version: '3.9'
|
|
|
|
- name: Clone s3-tests
|
|
run: |
|
|
git clone https://github.com/ceph/s3-tests.git
|
|
cd s3-tests
|
|
sudo apt-get update -qq
|
|
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
|
pip install -r requirements.txt
|
|
pip install tox
|
|
pip install -e .
|
|
|
|
- name: Run S3 CORS tests
|
|
timeout-minutes: 10
|
|
shell: bash
|
|
run: |
|
|
cd weed
|
|
go install -buildvcs=false
|
|
set -x
|
|
# Create clean data directory for this test run
|
|
export WEED_DATA_DIR="/tmp/seaweedfs-cors-test-$(date +%s)"
|
|
mkdir -p "$WEED_DATA_DIR"
|
|
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
|
|
-dir="$WEED_DATA_DIR" \
|
|
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
|
|
-volume.max=100 -volume.preStopSeconds=1 \
|
|
-master.port=9335 -volume.port=8082 -filer.port=8890 -s3.port=8002 -metricsPort=9326 \
|
|
-s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
|
|
pid=$!
|
|
|
|
# Wait for all SeaweedFS components to be ready
|
|
echo "Waiting for SeaweedFS components to start..."
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:9335/cluster/status > /dev/null 2>&1; then
|
|
echo "Master server is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for master server... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:8082/status > /dev/null 2>&1; then
|
|
echo "Volume server is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for volume server... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:8890/ > /dev/null 2>&1; then
|
|
echo "Filer is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for filer... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:8002/ > /dev/null 2>&1; then
|
|
echo "S3 server is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for S3 server... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
echo "All SeaweedFS components are ready!"
|
|
cd ../s3-tests
|
|
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests/functional/test_s3.py
|
|
# Create and update s3tests.conf to use port 8002
|
|
cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-cors.conf
|
|
sed -i 's/port = 8000/port = 8002/g' ../docker/compose/s3tests-cors.conf
|
|
sed -i 's/:8000/:8002/g' ../docker/compose/s3tests-cors.conf
|
|
sed -i 's/localhost:8000/localhost:8002/g' ../docker/compose/s3tests-cors.conf
|
|
sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8002/g' ../docker/compose/s3tests-cors.conf
|
|
export S3TEST_CONF=../docker/compose/s3tests-cors.conf
|
|
|
|
# Debug: Show the config file contents
|
|
echo "=== S3 Config File Contents ==="
|
|
cat ../docker/compose/s3tests-cors.conf
|
|
echo "=== End Config ==="
|
|
|
|
# Additional wait for S3-Filer integration to be fully ready
|
|
echo "Waiting additional 10 seconds for S3-Filer integration..."
|
|
sleep 10
|
|
|
|
# Test S3 connection before running tests
|
|
echo "Testing S3 connection..."
|
|
for i in {1..10}; do
|
|
if curl -s -f http://localhost:8002/ > /dev/null 2>&1; then
|
|
echo "S3 connection test successful"
|
|
break
|
|
fi
|
|
echo "S3 connection test failed, retrying... ($i/10)"
|
|
sleep 2
|
|
done
|
|
# Run CORS-specific tests from s3-tests suite
|
|
tox -- s3tests/functional/test_s3.py -k "cors" --tb=short || echo "No CORS tests found in s3-tests suite"
|
|
# If no specific CORS tests exist, run bucket configuration tests that include CORS
|
|
tox -- s3tests/functional/test_s3.py::test_put_bucket_cors || echo "No put_bucket_cors test found"
|
|
tox -- s3tests/functional/test_s3.py::test_get_bucket_cors || echo "No get_bucket_cors test found"
|
|
tox -- s3tests/functional/test_s3.py::test_delete_bucket_cors || echo "No delete_bucket_cors test found"
|
|
kill -9 $pid || true
|
|
# Clean up data directory
|
|
rm -rf "$WEED_DATA_DIR" || true
|
|
|
|
copy-tests:
|
|
name: SeaweedFS Custom S3 Copy tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Check out code into the Go module directory
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Set up Go 1.x
|
|
uses: actions/setup-go@v7
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Run SeaweedFS Custom S3 Copy tests
|
|
timeout-minutes: 10
|
|
shell: bash
|
|
run: |
|
|
cd weed
|
|
go install -buildvcs=false
|
|
# Create clean data directory for this test run
|
|
export WEED_DATA_DIR="/tmp/seaweedfs-copy-test-$(date +%s)"
|
|
mkdir -p "$WEED_DATA_DIR"
|
|
set -x
|
|
# Each test bucket is its own collection and grows 7 volumes; the suite runs
|
|
# faster than the heartbeat that returns slots from the deleted collections.
|
|
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
|
|
-dir="$WEED_DATA_DIR" \
|
|
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
|
|
-volume.max=300 -volume.preStopSeconds=1 \
|
|
-master.port=9336 -volume.port=8083 -filer.port=8891 -s3.port=8003 -metricsPort=9327 \
|
|
-s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
|
|
pid=$!
|
|
|
|
# Wait for all SeaweedFS components to be ready
|
|
echo "Waiting for SeaweedFS components to start..."
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:9336/cluster/status > /dev/null 2>&1; then
|
|
echo "Master server is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for master server... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:8083/status > /dev/null 2>&1; then
|
|
echo "Volume server is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for volume server... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:8891/ > /dev/null 2>&1; then
|
|
echo "Filer is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for filer... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:8003/ > /dev/null 2>&1; then
|
|
echo "S3 server is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for S3 server... ($i/30)"
|
|
sleep 2
|
|
done
|
|
|
|
echo "All SeaweedFS components are ready!"
|
|
cd ../test/s3/copying
|
|
# Patch Go tests to use the correct S3 endpoint (port 8003)
|
|
sed -i 's/http:\/\/127\.0\.0\.1:8000/http:\/\/127.0.0.1:8003/g' s3_copying_test.go
|
|
|
|
# Debug: Show what endpoint the Go tests will use
|
|
echo "=== Go Test Configuration ==="
|
|
grep -n "127.0.0.1" s3_copying_test.go || echo "No IP configuration found"
|
|
echo "=== End Configuration ==="
|
|
|
|
# Additional wait for S3-Filer integration to be fully ready
|
|
echo "Waiting additional 10 seconds for S3-Filer integration..."
|
|
sleep 10
|
|
|
|
# Test S3 connection before running tests
|
|
echo "Testing S3 connection..."
|
|
for i in {1..10}; do
|
|
if curl -s -f http://localhost:8003/ > /dev/null 2>&1; then
|
|
echo "S3 connection test successful"
|
|
break
|
|
fi
|
|
echo "S3 connection test failed, retrying... ($i/10)"
|
|
sleep 2
|
|
done
|
|
|
|
MASTER_ENDPOINT="http://127.0.0.1:9336" go test -v
|
|
kill -9 $pid || true
|
|
# Clean up data directory
|
|
rm -rf "$WEED_DATA_DIR" || true
|
|
|
|
sql-store-tests:
|
|
name: Basic S3 tests (SQL store)
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 15
|
|
steps:
|
|
- name: Check out code into the Go module directory
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Set up Go 1.x
|
|
uses: actions/setup-go@v7
|
|
with:
|
|
go-version-file: 'go.mod'
|
|
id: go
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v7
|
|
with:
|
|
python-version: '3.9'
|
|
|
|
- name: Clone s3-tests
|
|
run: |
|
|
git clone https://github.com/ceph/s3-tests.git
|
|
cd s3-tests
|
|
sudo apt-get update -qq
|
|
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
|
pip install -r requirements.txt
|
|
pip install tox
|
|
pip install -e .
|
|
|
|
- name: Run Ceph S3 tests with SQL store
|
|
timeout-minutes: 15
|
|
shell: bash
|
|
run: |
|
|
cd weed
|
|
|
|
# Debug: Check for port conflicts before starting
|
|
echo "=== Pre-start Port Check ==="
|
|
netstat -tulpn | grep -E "(9337|8085|8892|8004|9328)" || echo "Ports are free"
|
|
|
|
# Kill any existing weed processes that might interfere
|
|
echo "=== Cleanup existing processes ==="
|
|
pkill -f weed || echo "No weed processes found"
|
|
|
|
# More aggressive port cleanup using multiple methods
|
|
for port in 9337 8085 8892 8004 9328; do
|
|
echo "Cleaning port $port..."
|
|
|
|
# Method 1: lsof
|
|
pid=$(lsof -ti :$port 2>/dev/null || echo "")
|
|
if [ -n "$pid" ]; then
|
|
echo "Found process $pid using port $port (via lsof)"
|
|
kill -9 $pid 2>/dev/null || echo "Failed to kill $pid"
|
|
fi
|
|
|
|
# Method 2: netstat + ps (for cases where lsof fails)
|
|
netstat_pids=$(netstat -tlnp 2>/dev/null | grep ":$port " | awk '{print $7}' | cut -d'/' -f1 | grep -v '^-$' || echo "")
|
|
for npid in $netstat_pids; do
|
|
if [ -n "$npid" ] && [ "$npid" != "-" ]; then
|
|
echo "Found process $npid using port $port (via netstat)"
|
|
kill -9 $npid 2>/dev/null || echo "Failed to kill $npid"
|
|
fi
|
|
done
|
|
|
|
# Method 3: fuser (if available)
|
|
if command -v fuser >/dev/null 2>&1; then
|
|
fuser -k ${port}/tcp 2>/dev/null || echo "No process found via fuser for port $port"
|
|
fi
|
|
|
|
sleep 1
|
|
done
|
|
|
|
# Wait for ports to be released
|
|
sleep 5
|
|
|
|
echo "=== Post-cleanup Port Check ==="
|
|
netstat -tulpn | grep -E "(9337|8085|8892|8004|9328)" || echo "All ports are now free"
|
|
|
|
# If any ports are still in use, fail fast
|
|
if netstat -tulpn | grep -E "(9337|8085|8892|8004|9328)" >/dev/null 2>&1; then
|
|
echo "❌ ERROR: Some ports are still in use after aggressive cleanup!"
|
|
echo "=== Detailed Port Analysis ==="
|
|
for port in 9337 8085 8892 8004 9328; do
|
|
echo "Port $port:"
|
|
netstat -tlnp 2>/dev/null | grep ":$port " || echo " Not in use"
|
|
lsof -i :$port 2>/dev/null || echo " No lsof info"
|
|
done
|
|
exit 1
|
|
fi
|
|
|
|
go install -tags "sqlite s3tests" -buildvcs=false
|
|
# Create clean data directory for this test run with unique timestamp and process ID
|
|
export WEED_DATA_DIR="/tmp/seaweedfs-sql-test-$(date +%s)-$$"
|
|
mkdir -p "$WEED_DATA_DIR"
|
|
chmod 777 "$WEED_DATA_DIR"
|
|
|
|
# SQLite-specific configuration
|
|
export WEED_LEVELDB2_ENABLED="false"
|
|
export WEED_SQLITE_ENABLED="true"
|
|
export WEED_SQLITE_DBFILE="$WEED_DATA_DIR/filer.db"
|
|
|
|
echo "=== SQL Store Configuration ==="
|
|
echo "Data Dir: $WEED_DATA_DIR"
|
|
echo "SQLite DB: $WEED_SQLITE_DBFILE"
|
|
echo "LEVELDB2_ENABLED: $WEED_LEVELDB2_ENABLED"
|
|
echo "SQLITE_ENABLED: $WEED_SQLITE_ENABLED"
|
|
|
|
set -x
|
|
weed -v 1 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
|
|
-dir="$WEED_DATA_DIR" \
|
|
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
|
|
-volume.max=100 -volume.preStopSeconds=1 \
|
|
-master.port=9337 -volume.port=8085 -filer.port=8892 -s3.port=8004 -metricsPort=9328 \
|
|
-s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \
|
|
-master.peers=none \
|
|
> /tmp/seaweedfs-sql-server.log 2>&1 &
|
|
pid=$!
|
|
|
|
echo "=== Server started with PID: $pid ==="
|
|
|
|
# Wait for all SeaweedFS components to be ready
|
|
echo "Waiting for SeaweedFS components to start..."
|
|
|
|
# Check if server process is still alive before waiting
|
|
if ! kill -0 $pid 2>/dev/null; then
|
|
echo "❌ Server process died immediately after start"
|
|
echo "=== Immediate Log Check ==="
|
|
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null || echo "No log available"
|
|
exit 1
|
|
fi
|
|
|
|
sleep 5 # Give SQLite more time to initialize
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:9337/cluster/status > /dev/null 2>&1; then
|
|
echo "Master server is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for master server... ($i/30)"
|
|
# Check if server process is still alive
|
|
if ! kill -0 $pid 2>/dev/null; then
|
|
echo "❌ Server process died while waiting for master"
|
|
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
|
|
exit 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:8085/status > /dev/null 2>&1; then
|
|
echo "Volume server is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for volume server... ($i/30)"
|
|
if ! kill -0 $pid 2>/dev/null; then
|
|
echo "❌ Server process died while waiting for volume"
|
|
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
|
|
exit 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:8892/ > /dev/null 2>&1; then
|
|
echo "Filer (SQLite) is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for filer (SQLite)... ($i/30)"
|
|
if ! kill -0 $pid 2>/dev/null; then
|
|
echo "❌ Server process died while waiting for filer"
|
|
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
|
|
exit 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
# Extra wait for SQLite filer to fully initialize
|
|
echo "Giving SQLite filer extra time to initialize..."
|
|
sleep 5
|
|
|
|
for i in {1..30}; do
|
|
if curl -s http://localhost:8004/ > /dev/null 2>&1; then
|
|
echo "S3 server is ready"
|
|
break
|
|
fi
|
|
echo "Waiting for S3 server... ($i/30)"
|
|
if ! kill -0 $pid 2>/dev/null; then
|
|
echo "❌ Server process died while waiting for S3"
|
|
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
|
|
exit 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
echo "All SeaweedFS components are ready!"
|
|
cd ../s3-tests
|
|
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests/functional/test_s3.py
|
|
# The suite expects RGW's 400 InvalidPart for a partNumber past the last
|
|
# part; AWS answers 416 InvalidPartNumber, which is what we return.
|
|
sed -i "/# request PartNumber out of range/,+4{s/assert status == 400/assert status == 416/; s/assert error_code == 'InvalidPart'/assert error_code == 'InvalidPartNumber'/}" s3tests/functional/test_s3.py
|
|
# Create and update s3tests.conf to use port 8004
|
|
cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-sql.conf
|
|
sed -i 's/port = 8000/port = 8004/g' ../docker/compose/s3tests-sql.conf
|
|
sed -i 's/:8000/:8004/g' ../docker/compose/s3tests-sql.conf
|
|
sed -i 's/localhost:8000/localhost:8004/g' ../docker/compose/s3tests-sql.conf
|
|
sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8004/g' ../docker/compose/s3tests-sql.conf
|
|
export S3TEST_CONF=../docker/compose/s3tests-sql.conf
|
|
|
|
# Debug: Show the config file contents
|
|
echo "=== S3 Config File Contents ==="
|
|
cat ../docker/compose/s3tests-sql.conf
|
|
echo "=== End Config ==="
|
|
|
|
# Additional wait for S3-Filer integration to be fully ready
|
|
echo "Waiting additional 10 seconds for S3-Filer integration..."
|
|
sleep 10
|
|
|
|
# Test S3 connection before running tests
|
|
echo "Testing S3 connection..."
|
|
|
|
# Debug: Check if SeaweedFS processes are running
|
|
echo "=== Process Status ==="
|
|
ps aux | grep -E "(weed|seaweedfs)" | grep -v grep || echo "No SeaweedFS processes found"
|
|
|
|
# Debug: Check port status
|
|
echo "=== Port Status ==="
|
|
netstat -tulpn | grep -E "(8004|9337|8085|8892)" || echo "Ports not found"
|
|
|
|
# Debug: Check server logs
|
|
echo "=== Recent Server Logs ==="
|
|
echo "--- SQL Server Log ---"
|
|
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null || echo "No SQL server log found"
|
|
echo "--- Other Logs ---"
|
|
ls -la /tmp/seaweedfs-*.log 2>/dev/null || echo "No other log files found"
|
|
|
|
for i in {1..10}; do
|
|
if curl -s -f http://localhost:8004/ > /dev/null 2>&1; then
|
|
echo "S3 connection test successful"
|
|
break
|
|
fi
|
|
echo "S3 connection test failed, retrying... ($i/10)"
|
|
|
|
# Debug: Try different HTTP methods
|
|
echo "Debug: Testing different endpoints..."
|
|
curl -s -I http://localhost:8004/ || echo "HEAD request failed"
|
|
curl -s http://localhost:8004/status || echo "Status endpoint failed"
|
|
|
|
sleep 2
|
|
done
|
|
|
|
# Spawn the lifecycle worker (see basic-tests block for context).
|
|
LC_LOG=/tmp/lifecycle-worker-sql.log
|
|
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18004 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
|
|
| weed shell -debug -master=localhost:9337 \
|
|
> "$LC_LOG" 2>&1 &
|
|
lc_pid=$!
|
|
sleep 2
|
|
if ! kill -0 "$lc_pid" 2>/dev/null; then
|
|
echo "lifecycle worker died on startup"
|
|
tail -50 "$LC_LOG" 2>/dev/null || true
|
|
exit 1
|
|
fi
|
|
echo "lifecycle worker pid=$lc_pid"
|
|
|
|
cleanup() {
|
|
status=$?
|
|
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
|
|
# bash fallback if it ignores TERM. Reading the log AFTER the
|
|
# graceful-stop window catches the bootstrap walker's progress.
|
|
kill -TERM "$lc_pid" 2>/dev/null || true
|
|
kill -TERM "$pid" 2>/dev/null || true
|
|
sleep 1
|
|
if [ "$status" -ne 0 ]; then
|
|
echo "=== lifecycle worker log (tail) ==="
|
|
tail -200 "$LC_LOG" 2>/dev/null || true
|
|
fi
|
|
kill -9 "$lc_pid" 2>/dev/null || true
|
|
kill -9 "$pid" 2>/dev/null || true
|
|
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
tox -- \
|
|
s3tests/functional/test_s3.py::test_bucket_list_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_list_distinct \
|
|
s3tests/functional/test_s3.py::test_bucket_list_many \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_many \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_encoding_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_list_encoding_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix_ends_with_delimiter \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_alt \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_alt \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_prefix_underscore \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_percentage \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_percentage \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_whitespace \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_whitespace \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_dot \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_dot \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_unreadable \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_none \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_none \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_delimiter_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_list_delimiter_not_skip_special \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_alt \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_alt \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_delimiter_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_defaultempty \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_fetchowner_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_basic \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_alt \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_alt \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_none \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_none \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_not_exist \
|
|
s3tests/functional/test_s3.py::test_bucket_list_prefix_unreadable \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_prefix_unreadable \
|
|
s3tests/functional/test_s3.py::test_bucket_list_maxkeys_one \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_one \
|
|
s3tests/functional/test_s3.py::test_bucket_list_maxkeys_zero \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \
|
|
s3tests/functional/test_s3.py::test_bucket_list_maxkeys_none \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_maxkeys_none \
|
|
s3tests/functional/test_s3.py::test_bucket_list_unordered \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_unordered \
|
|
s3tests/functional/test_s3.py::test_bucket_list_maxkeys_invalid \
|
|
s3tests/functional/test_s3.py::test_bucket_list_marker_none \
|
|
s3tests/functional/test_s3.py::test_bucket_list_marker_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_continuationtoken_empty \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_continuationtoken \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \
|
|
s3tests/functional/test_s3.py::test_bucket_list_marker_unreadable \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_startafter_unreadable \
|
|
s3tests/functional/test_s3.py::test_bucket_list_marker_not_in_list \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_startafter_not_in_list \
|
|
s3tests/functional/test_s3.py::test_bucket_list_marker_after_list \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_startafter_after_list \
|
|
s3tests/functional/test_s3.py::test_bucket_list_return_data \
|
|
s3tests/functional/test_s3.py::test_bucket_list_objects_anonymous \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_objects_anonymous \
|
|
s3tests/functional/test_s3.py::test_bucket_list_objects_anonymous_fail \
|
|
s3tests/functional/test_s3.py::test_bucket_listv2_objects_anonymous_fail \
|
|
s3tests/functional/test_s3.py::test_bucket_list_long_name \
|
|
s3tests/functional/test_s3.py::test_bucket_list_special_prefix \
|
|
s3tests/functional/test_s3.py::test_bucket_delete_notexist \
|
|
s3tests/functional/test_s3.py::test_bucket_create_delete \
|
|
s3tests/functional/test_s3.py::test_object_read_not_exist \
|
|
s3tests/functional/test_s3.py::test_multi_object_delete \
|
|
s3tests/functional/test_s3.py::test_multi_objectv2_delete \
|
|
s3tests/functional/test_s3.py::test_object_head_zero_bytes \
|
|
s3tests/functional/test_s3.py::test_object_write_check_etag \
|
|
s3tests/functional/test_s3.py::test_object_write_cache_control \
|
|
s3tests/functional/test_s3.py::test_object_write_expires \
|
|
s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
|
|
s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
|
|
s3tests/functional/test_s3.py::test_object_write_file \
|
|
s3tests/functional/test_s3.py::test_post_object_invalid_date_format \
|
|
s3tests/functional/test_s3.py::test_post_object_no_key_specified \
|
|
s3tests/functional/test_s3.py::test_post_object_missing_signature \
|
|
s3tests/functional/test_s3.py::test_post_object_condition_is_case_sensitive \
|
|
s3tests/functional/test_s3.py::test_post_object_expires_is_case_sensitive \
|
|
s3tests/functional/test_s3.py::test_post_object_missing_expires_condition \
|
|
s3tests/functional/test_s3.py::test_post_object_missing_conditions_list \
|
|
s3tests/functional/test_s3.py::test_post_object_upload_size_limit_exceeded \
|
|
s3tests/functional/test_s3.py::test_post_object_missing_content_length_argument \
|
|
s3tests/functional/test_s3.py::test_post_object_invalid_content_length_argument \
|
|
s3tests/functional/test_s3.py::test_post_object_upload_size_below_minimum \
|
|
s3tests/functional/test_s3.py::test_post_object_empty_conditions \
|
|
s3tests/functional/test_s3.py::test_get_object_ifmatch_good \
|
|
s3tests/functional/test_s3.py::test_get_object_ifnonematch_good \
|
|
s3tests/functional/test_s3.py::test_get_object_ifmatch_failed \
|
|
s3tests/functional/test_s3.py::test_get_object_ifnonematch_failed \
|
|
s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
|
|
s3tests/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
|
|
s3tests/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
|
|
s3tests/functional/test_s3.py::test_bucket_head \
|
|
s3tests/functional/test_s3.py::test_bucket_head_notexist \
|
|
s3tests/functional/test_s3.py::test_object_raw_authenticated \
|
|
s3tests/functional/test_s3.py::test_object_raw_authenticated_bucket_acl \
|
|
s3tests/functional/test_s3.py::test_object_raw_authenticated_object_acl \
|
|
s3tests/functional/test_s3.py::test_object_raw_authenticated_object_gone \
|
|
s3tests/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_range_zero \
|
|
s3tests/functional/test_s3.py::test_object_anon_put \
|
|
s3tests/functional/test_s3.py::test_object_put_authenticated \
|
|
s3tests/functional/test_s3.py::test_bucket_recreate_overwrite_acl \
|
|
s3tests/functional/test_s3.py::test_bucket_recreate_new_acl \
|
|
s3tests/functional/test_s3.py::test_buckets_create_then_list \
|
|
s3tests/functional/test_s3.py::test_buckets_list_ctime \
|
|
s3tests/functional/test_s3.py::test_list_buckets_invalid_auth \
|
|
s3tests/functional/test_s3.py::test_list_buckets_bad_auth \
|
|
s3tests/functional/test_s3.py::test_bucket_create_naming_good_contains_period \
|
|
s3tests/functional/test_s3.py::test_bucket_create_naming_good_contains_hyphen \
|
|
s3tests/functional/test_s3.py::test_bucket_list_special_prefix \
|
|
s3tests/functional/test_s3.py::test_object_copy_zero_size \
|
|
s3tests/functional/test_s3.py::test_object_copy_same_bucket \
|
|
s3tests/functional/test_s3.py::test_object_copy_to_itself \
|
|
s3tests/functional/test_s3.py::test_object_copy_diff_bucket \
|
|
s3tests/functional/test_s3.py::test_object_copy_canned_acl \
|
|
s3tests/functional/test_s3.py::test_object_copy_bucket_not_found \
|
|
s3tests/functional/test_s3.py::test_object_copy_key_not_found \
|
|
s3tests/functional/test_s3.py::test_multipart_copy_small \
|
|
s3tests/functional/test_s3.py::test_multipart_copy_without_range \
|
|
s3tests/functional/test_s3.py::test_multipart_copy_special_names \
|
|
s3tests/functional/test_s3.py::test_multipart_copy_multiple_sizes \
|
|
s3tests/functional/test_s3.py::test_multipart_get_part \
|
|
s3tests/functional/test_s3.py::test_multipart_upload \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_empty \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_multiple_sizes \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_contents \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_size_too_small \
|
|
s3tests/functional/test_s3.py::test_multipart_resend_first_finishes_last \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_resend_part \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_missing_part \
|
|
s3tests/functional/test_s3.py::test_multipart_upload_incorrect_etag \
|
|
s3tests/functional/test_s3.py::test_abort_multipart_upload \
|
|
s3tests/functional/test_s3.py::test_list_multipart_upload \
|
|
s3tests/functional/test_s3.py::test_atomic_read_1mb \
|
|
s3tests/functional/test_s3.py::test_atomic_read_4mb \
|
|
s3tests/functional/test_s3.py::test_atomic_read_8mb \
|
|
s3tests/functional/test_s3.py::test_atomic_write_1mb \
|
|
s3tests/functional/test_s3.py::test_atomic_write_4mb \
|
|
s3tests/functional/test_s3.py::test_atomic_write_8mb \
|
|
s3tests/functional/test_s3.py::test_atomic_dual_write_1mb \
|
|
s3tests/functional/test_s3.py::test_atomic_dual_write_4mb \
|
|
s3tests/functional/test_s3.py::test_atomic_dual_write_8mb \
|
|
s3tests/functional/test_s3.py::test_atomic_multipart_upload_write \
|
|
s3tests/functional/test_s3.py::test_ranged_request_response_code \
|
|
s3tests/functional/test_s3.py::test_ranged_big_request_response_code \
|
|
s3tests/functional/test_s3.py::test_ranged_request_skip_leading_bytes_response_code \
|
|
s3tests/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \
|
|
s3tests/functional/test_s3.py::test_copy_object_ifmatch_good \
|
|
s3tests/functional/test_s3.py::test_copy_object_ifnonematch_failed \
|
|
s3tests/functional/test_s3.py::test_copy_object_ifmatch_failed \
|
|
s3tests/functional/test_s3.py::test_copy_object_ifnonematch_good \
|
|
s3tests/functional/test_s3.py::test_lifecycle_set \
|
|
s3tests/functional/test_s3.py::test_lifecycle_get \
|
|
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
|
|
s3tests/functional/test_s3.py::test_lifecycle_expiration \
|
|
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
|
|
# cleanup() trap handles worker/server kill + data dir wipe.
|
|
|
|
|