Compare commits

..
Author SHA1 Message Date
chrislu e7d12a1c04 rename 2025-07-19 00:46:35 -07:00
chrislu 11fb65571f refactor 2025-07-19 00:42:43 -07:00
chrislu 6c2a11f1d6 Merge branch 'master' into fix-GetObjectLockConfigurationHandler 2025-07-19 00:42:36 -07:00
chrislu 851011a32b address comments 2025-07-18 22:16:15 -07:00
chrislu a9a9c732b9 reduce logs 2025-07-18 21:51:13 -07:00
Chris LuandCopilot 6328141019 Update weed/s3api/s3api_object_retention.go
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-07-18 21:46:59 -07:00
chrislu b508e00de7 fix test 2025-07-18 21:40:04 -07:00
chrislu e486817499 fix tests 2025-07-18 21:32:38 -07:00
chrislu aec8542b6e fix test itself 2025-07-18 21:17:42 -07:00
chrislu 8fb9df72ae fix tests 2025-07-18 20:55:12 -07:00
chrislu af25f92e36 fix tests 2025-07-18 20:52:03 -07:00
chrislu 66829fa7c8 fix tests 2025-07-18 20:46:56 -07:00
chrislu 5c24bfc795 fix test_object_lock_delete_object_with_retention_and_marker 2025-07-18 20:43:09 -07:00
chrislu 1d1c717493 refactor 2025-07-18 20:16:04 -07:00
chrislu 7661449f82 fix tests 2025-07-18 20:10:16 -07:00
chrislu a5a6b5a4d4 fix test_object_lock_delete_multipart_object_with_legal_hold_on 2025-07-18 19:52:51 -07:00
chrislu 8121fdeec3 fix many issues 2025-07-18 19:40:12 -07:00
chrislu f63f40a3d4 fixes 2025-07-18 18:04:10 -07:00
chrislu 2d6532d80e fix test_object_lock_put_obj_lock_invalid_days 2025-07-18 17:51:14 -07:00
chrislu dce0672ae0 Update s3tests.conf 2025-07-18 17:19:15 -07:00
chrislu 40d9f4fa9c add error mapping 2025-07-18 17:08:59 -07:00
chrislu 8cc99ca121 fixes 2025-07-18 17:06:46 -07:00
chrislu c5628b000b fix tests 2025-07-18 16:47:25 -07:00
chrislu f419271299 pass tests 2025-07-18 16:38:24 -07:00
chrislu a0ab227e07 fixes 2025-07-18 16:25:45 -07:00
chrislu 2ad4a924be fix tests 2025-07-18 16:23:15 -07:00
chrislu 1e1806c7a1 fixes
✅ Include VersionId in multipart upload completion responses when versioning is enabled
✅ Block retention mode changes (GOVERNANCE ↔ COMPLIANCE) without bypass permissions
✅ Handle all object lock validation errors consistently with proper error codes
✅ Pass the remaining object lock tests
2025-07-18 16:17:36 -07:00
chrislu eaaef569ce fixes
✅ Return 409 (Conflict) with InvalidBucketState for bucket-level object lock configuration operations on buckets without object lock enabled
✅ Allow increasing retention periods and overriding retention with same/later dates
✅ Only block decreasing retention periods without proper bypass permissions
✅ Handle all object lock validation errors consistently with proper error codes
2025-07-18 16:11:14 -07:00
chrislu 1807f4e94d fixes
✅ Return MalformedXML when both Days and Years are specified in the same retention configuration
✅ Return 400 (Bad Request) with InvalidRequest when object lock operations are attempted on buckets without object lock enabled
✅ Handle all object lock validation errors consistently with proper error codes
2025-07-18 16:03:06 -07:00
chrislu 7203c78e4d constants and fixes
✅ Return InvalidRetentionPeriod for invalid retention values (0 days, negative years)
✅ Return ObjectLockConfigurationNotFoundError when object lock configuration doesn't exist
✅ Handle all object lock validation errors consistently with proper error codes
2025-07-18 15:58:07 -07:00
chrislu 1bc503530a fixes
With these final fixes, the s3-tests should now:
✅ Return MalformedXML for ObjectLockEnabled: 'Disabled'
✅ Return MalformedXML when both Days and Years are specified in retention configuration
✅ Return InvalidBucketState (409 Conflict) when trying to suspend versioning on buckets with object lock enabled
✅ Handle all object lock validation errors consistently with proper error codes
2025-07-18 15:52:28 -07:00
chrislu 9f88fd2ea5 fixes
With these comprehensive fixes, the s3-tests should now:
✅ Return InvalidBucketState (409 Conflict) for object lock operations on invalid buckets
✅ Return InvalidRetentionPeriod for invalid retention periods
✅ Return MalformedXML for malformed retention configurations
✅ Include VersionId in response headers when available
✅ Return proper HTTP status codes for all error conditions
✅ Handle all object lock validation errors consistently
The workflow should now pass significantly more object lock tests, bringing SeaweedFS's S3 object lock implementation much closer to AWS S3 compatibility standards.
2025-07-18 15:45:15 -07:00
chrislu 0123abe49b address test errors
With these fixes, the s3-tests should now:
✅ Return InvalidBucketState (409 Conflict) for object lock operations on invalid buckets
✅ Return MalformedXML for invalid retention configurations
✅ Include VersionId in response headers when available
✅ Return proper HTTP status codes (403 Forbidden for retention mode changes)
✅ Handle all object lock validation errors consistently
2025-07-18 15:39:30 -07:00
chrislu 4b92f03b65 address test errors 2025-07-18 15:33:44 -07:00
chrislu 1e7a3c09d4 Update s3tests.conf 2025-07-18 15:23:05 -07:00
chrislu ddb2105f00 Update s3tests.conf 2025-07-18 15:17:58 -07:00
chrislu bbcd19f6c6 Update s3tests.yml 2025-07-18 15:14:09 -07:00
chrislu fc2fcae543 Update s3tests.yml 2025-07-18 15:03:21 -07:00
chrislu 4801f641f1 install s3 tests 2025-07-18 15:01:25 -07:00
chrislu 770888df81 use marker 2025-07-18 14:54:15 -07:00
chrislu 1dfdd3c84d Merge branch 'master' into fix-GetObjectLockConfigurationHandler 2025-07-18 14:47:36 -07:00
chrislu b63855560a add s3 tests for object lock and retention 2025-07-18 14:46:32 -07:00
chrislu 9f8e0257b0 read entry once 2025-07-18 14:39:02 -07:00
chrislu e53cd29eb2 errorf 2025-07-18 02:14:19 -07:00
chrislu eeeb595346 change error style 2025-07-18 02:02:54 -07:00
Chris LuandCopilot 678f962ded Update weed/s3api/object_lock_utils.go
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-07-18 01:55:43 -07:00
Chris LuandCopilot a906358c59 Update test/s3/retention/s3_retention_test.go
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-07-18 01:54:36 -07:00
Chris LuandCopilot 12f34f965f Update weed/s3api/s3api_object_handlers_put.go
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-07-18 01:54:14 -07:00
chrislu 1db149ecc2 address comments 2025-07-18 01:49:44 -07:00
chrislu 59f8444a90 fix test 2025-07-18 01:42:03 -07:00
chrislu bcc7a74a25 use fields instead of xml 2025-07-18 01:39:23 -07:00
chrislu 0d4eeb67be admin ui and api handler are consistent now 2025-07-18 01:27:46 -07:00
chrislu 85e34df214 fix routing 2025-07-18 01:27:27 -07:00
chrislu bb05e6cade Update s3api_object_handlers_put.go 2025-07-18 00:53:20 -07:00
chrislu bc46214445 rename variable 2025-07-18 00:52:38 -07:00
chrislu 121f6ef2c0 avoid duplidated work 2025-07-18 00:48:56 -07:00
Chris Luandgemini-code-assist[bot] 22673fee76 Update weed/s3api/s3api_server.go
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
2025-07-18 00:47:24 -07:00
chrislu 3fbab20b83 refactor 2025-07-18 00:34:16 -07:00
chrislu b8cbe9b6b2 increase bucket config cache TTL 2025-07-18 00:26:58 -07:00
chrislu 8bb47f3c03 subscribe to bucket configuration changes 2025-07-18 00:25:18 -07:00
chrislu 7bec2ef94b cache and use bucket object lock config 2025-07-18 00:19:47 -07:00
chrislu f9cbed7ee1 fix GetObjectLockConfigurationHandler 2025-07-17 23:12:19 -07:00
22 changed files with 261 additions and 1565 deletions

No files matched your search

+29 -606
View File
@@ -14,10 +14,10 @@ permissions:
contents: read
jobs:
basic-s3-tests:
name: Basic S3 tests (KV store)
s3tests:
name: Ceph S3 tests
runs-on: ubuntu-22.04
timeout-minutes: 15
timeout-minutes: 30
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
@@ -41,7 +41,7 @@ jobs:
pip install tox
pip install -e .
- name: Run Basic S3 tests
- name: Run Ceph S3 tests with KV store
timeout-minutes: 15
env:
S3TEST_CONF: ../docker/compose/s3tests.conf
@@ -55,76 +55,13 @@ jobs:
mkdir -p "$WEED_DATA_DIR"
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9333 -volume.port=8080 -filer.port=8888 -s3.port=8000 -metricsPort=9324 \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
-volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
pid=$!
# Wait for all SeaweedFS components to be ready
echo "Waiting for SeaweedFS components to start..."
for i in {1..30}; do
if curl -s http://localhost:9333/cluster/status > /dev/null 2>&1; then
echo "Master server is ready"
break
fi
echo "Waiting for master server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8080/status > /dev/null 2>&1; then
echo "Volume server is ready"
break
fi
echo "Waiting for volume server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8888/ > /dev/null 2>&1; then
echo "Filer is ready"
break
fi
echo "Waiting for filer... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8000/ > /dev/null 2>&1; then
echo "S3 server is ready"
break
fi
echo "Waiting for S3 server... ($i/30)"
sleep 2
done
echo "All SeaweedFS components are ready!"
sleep 10
cd ../s3-tests
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
# Debug: Show the config file contents
echo "=== S3 Config File Contents ==="
cat ../docker/compose/s3tests.conf
echo "=== End Config ==="
# Additional wait for S3-Filer integration to be fully ready
echo "Waiting additional 10 seconds for S3-Filer integration..."
sleep 10
# Test S3 connection before running tests
echo "Testing S3 connection..."
for i in {1..10}; do
if curl -s -f http://localhost:8000/ > /dev/null 2>&1; then
echo "S3 connection test successful"
break
fi
echo "S3 connection test failed, retrying... ($i/10)"
sleep 2
done
echo "✅ S3 server is responding, starting tests..."
tox -- \
s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \
s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \
@@ -283,264 +220,33 @@ jobs:
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
versioning-tests:
name: S3 Versioning & Object Lock tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: Set up Go 1.x
uses: actions/setup-go@v5.5.0
with:
go-version-file: 'go.mod'
id: go
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Clone s3-tests
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Run S3 Object Lock, Retention, and Versioning tests
- name: Run S3 Object Lock and Retention tests
timeout-minutes: 15
env:
S3TEST_CONF: ../docker/compose/s3tests.conf
shell: bash
run: |
cd weed
go install -buildvcs=false
set -x
# Create clean data directory for this test run
export WEED_DATA_DIR="/tmp/seaweedfs-objectlock-versioning-$(date +%s)"
export WEED_DATA_DIR="/tmp/seaweedfs-objectlock-$(date +%s)"
mkdir -p "$WEED_DATA_DIR"
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9334 -volume.port=8081 -filer.port=8889 -s3.port=8001 -metricsPort=9325 \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
-volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
pid=$!
# Wait for all SeaweedFS components to be ready
echo "Waiting for SeaweedFS components to start..."
for i in {1..30}; do
if curl -s http://localhost:9334/cluster/status > /dev/null 2>&1; then
echo "Master server is ready"
break
fi
echo "Waiting for master server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8081/status > /dev/null 2>&1; then
echo "Volume server is ready"
break
fi
echo "Waiting for volume server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8889/ > /dev/null 2>&1; then
echo "Filer is ready"
break
fi
echo "Waiting for filer... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8001/ > /dev/null 2>&1; then
echo "S3 server is ready"
break
fi
echo "Waiting for S3 server... ($i/30)"
sleep 2
done
echo "All SeaweedFS components are ready!"
sleep 10
cd ../s3-tests
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
# Fix bucket creation conflicts in versioning tests by replacing _create_objects calls
sed -i 's/bucket_name = _create_objects(bucket_name=bucket_name,keys=key_names)/# Use the existing bucket for object creation\n client = get_client()\n for key in key_names:\n client.put_object(Bucket=bucket_name, Body=key, Key=key)/' s3tests_boto3/functional/test_s3.py
sed -i 's/bucket = _create_objects(bucket_name=bucket_name, keys=key_names)/# Use the existing bucket for object creation\n client = get_client()\n for key in key_names:\n client.put_object(Bucket=bucket_name, Body=key, Key=key)/' s3tests_boto3/functional/test_s3.py
# Create and update s3tests.conf to use port 8001
cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-versioning.conf
sed -i 's/port = 8000/port = 8001/g' ../docker/compose/s3tests-versioning.conf
sed -i 's/:8000/:8001/g' ../docker/compose/s3tests-versioning.conf
sed -i 's/localhost:8000/localhost:8001/g' ../docker/compose/s3tests-versioning.conf
sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8001/g' ../docker/compose/s3tests-versioning.conf
export S3TEST_CONF=../docker/compose/s3tests-versioning.conf
# Debug: Show the config file contents
echo "=== S3 Config File Contents ==="
cat ../docker/compose/s3tests-versioning.conf
echo "=== End Config ==="
# Additional wait for S3-Filer integration to be fully ready
echo "Waiting additional 10 seconds for S3-Filer integration..."
sleep 10
# Test S3 connection before running tests
echo "Testing S3 connection..."
for i in {1..10}; do
if curl -s -f http://localhost:8001/ > /dev/null 2>&1; then
echo "S3 connection test successful"
break
fi
echo "S3 connection test failed, retrying... ($i/10)"
sleep 2
done
tox -- s3tests_boto3/functional/test_s3.py -k "object_lock or (versioning and not test_versioning_obj_suspend_versions and not test_bucket_list_return_data_versioning and not test_versioning_concurrent_multi_object_delete)" --tb=short
# Run object lock tests by pattern matching test names
tox -- -k "object_lock" --tb=short
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
cors-tests:
name: S3 CORS tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: Set up Go 1.x
uses: actions/setup-go@v5.5.0
with:
go-version-file: 'go.mod'
id: go
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Clone s3-tests
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Run S3 CORS tests
timeout-minutes: 10
shell: bash
run: |
cd weed
go install -buildvcs=false
set -x
# Create clean data directory for this test run
export WEED_DATA_DIR="/tmp/seaweedfs-cors-test-$(date +%s)"
mkdir -p "$WEED_DATA_DIR"
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9335 -volume.port=8082 -filer.port=8890 -s3.port=8002 -metricsPort=9326 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
pid=$!
# Wait for all SeaweedFS components to be ready
echo "Waiting for SeaweedFS components to start..."
for i in {1..30}; do
if curl -s http://localhost:9335/cluster/status > /dev/null 2>&1; then
echo "Master server is ready"
break
fi
echo "Waiting for master server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8082/status > /dev/null 2>&1; then
echo "Volume server is ready"
break
fi
echo "Waiting for volume server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8890/ > /dev/null 2>&1; then
echo "Filer is ready"
break
fi
echo "Waiting for filer... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8002/ > /dev/null 2>&1; then
echo "S3 server is ready"
break
fi
echo "Waiting for S3 server... ($i/30)"
sleep 2
done
echo "All SeaweedFS components are ready!"
cd ../s3-tests
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
# Create and update s3tests.conf to use port 8002
cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-cors.conf
sed -i 's/port = 8000/port = 8002/g' ../docker/compose/s3tests-cors.conf
sed -i 's/:8000/:8002/g' ../docker/compose/s3tests-cors.conf
sed -i 's/localhost:8000/localhost:8002/g' ../docker/compose/s3tests-cors.conf
sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8002/g' ../docker/compose/s3tests-cors.conf
export S3TEST_CONF=../docker/compose/s3tests-cors.conf
# Debug: Show the config file contents
echo "=== S3 Config File Contents ==="
cat ../docker/compose/s3tests-cors.conf
echo "=== End Config ==="
# Additional wait for S3-Filer integration to be fully ready
echo "Waiting additional 10 seconds for S3-Filer integration..."
sleep 10
# Test S3 connection before running tests
echo "Testing S3 connection..."
for i in {1..10}; do
if curl -s -f http://localhost:8002/ > /dev/null 2>&1; then
echo "S3 connection test successful"
break
fi
echo "S3 connection test failed, retrying... ($i/10)"
sleep 2
done
# Run CORS-specific tests from s3-tests suite
tox -- s3tests_boto3/functional/test_s3.py -k "cors" --tb=short || echo "No CORS tests found in s3-tests suite"
# If no specific CORS tests exist, run bucket configuration tests that include CORS
tox -- s3tests_boto3/functional/test_s3.py::test_put_bucket_cors || echo "No put_bucket_cors test found"
tox -- s3tests_boto3/functional/test_s3.py::test_get_bucket_cors || echo "No get_bucket_cors test found"
tox -- s3tests_boto3/functional/test_s3.py::test_delete_bucket_cors || echo "No delete_bucket_cors test found"
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
copy-tests:
name: SeaweedFS Custom S3 Copy tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: Set up Go 1.x
uses: actions/setup-go@v5.5.0
with:
go-version-file: 'go.mod'
id: go
- name: Run SeaweedFS Custom S3 Copy tests
timeout-minutes: 10
shell: bash
@@ -553,322 +259,39 @@ jobs:
set -x
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9336 -volume.port=8083 -filer.port=8891 -s3.port=8003 -metricsPort=9327 \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
-volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
pid=$!
# Wait for all SeaweedFS components to be ready
echo "Waiting for SeaweedFS components to start..."
for i in {1..30}; do
if curl -s http://localhost:9336/cluster/status > /dev/null 2>&1; then
echo "Master server is ready"
break
fi
echo "Waiting for master server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8083/status > /dev/null 2>&1; then
echo "Volume server is ready"
break
fi
echo "Waiting for volume server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8891/ > /dev/null 2>&1; then
echo "Filer is ready"
break
fi
echo "Waiting for filer... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8003/ > /dev/null 2>&1; then
echo "S3 server is ready"
break
fi
echo "Waiting for S3 server... ($i/30)"
sleep 2
done
echo "All SeaweedFS components are ready!"
cd ../test/s3/copying
# Patch Go tests to use the correct S3 endpoint (port 8003)
sed -i 's/http:\/\/127\.0\.0\.1:8000/http:\/\/127.0.0.1:8003/g' s3_copying_test.go
# Debug: Show what endpoint the Go tests will use
echo "=== Go Test Configuration ==="
grep -n "127.0.0.1" s3_copying_test.go || echo "No IP configuration found"
echo "=== End Configuration ==="
# Additional wait for S3-Filer integration to be fully ready
echo "Waiting additional 10 seconds for S3-Filer integration..."
sleep 10
# Test S3 connection before running tests
echo "Testing S3 connection..."
for i in {1..10}; do
if curl -s -f http://localhost:8003/ > /dev/null 2>&1; then
echo "S3 connection test successful"
break
fi
echo "S3 connection test failed, retrying... ($i/10)"
sleep 2
done
cd ../test/s3/copying
go test -v
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
sql-store-tests:
name: Basic S3 tests (SQL store)
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: Set up Go 1.x
uses: actions/setup-go@v5.5.0
with:
go-version-file: 'go.mod'
id: go
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Clone s3-tests
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Run Ceph S3 tests with SQL store
timeout-minutes: 15
env:
S3TEST_CONF: ../docker/compose/s3tests.conf
shell: bash
run: |
cd weed
# Debug: Check for port conflicts before starting
echo "=== Pre-start Port Check ==="
netstat -tulpn | grep -E "(9337|8085|8892|8004|9328)" || echo "Ports are free"
# Kill any existing weed processes that might interfere
echo "=== Cleanup existing processes ==="
pkill -f weed || echo "No weed processes found"
# More aggressive port cleanup using multiple methods
for port in 9337 8085 8892 8004 9328; do
echo "Cleaning port $port..."
# Method 1: lsof
pid=$(lsof -ti :$port 2>/dev/null || echo "")
if [ -n "$pid" ]; then
echo "Found process $pid using port $port (via lsof)"
kill -9 $pid 2>/dev/null || echo "Failed to kill $pid"
fi
# Method 2: netstat + ps (for cases where lsof fails)
netstat_pids=$(netstat -tlnp 2>/dev/null | grep ":$port " | awk '{print $7}' | cut -d'/' -f1 | grep -v '^-$' || echo "")
for npid in $netstat_pids; do
if [ -n "$npid" ] && [ "$npid" != "-" ]; then
echo "Found process $npid using port $port (via netstat)"
kill -9 $npid 2>/dev/null || echo "Failed to kill $npid"
fi
done
# Method 3: fuser (if available)
if command -v fuser >/dev/null 2>&1; then
fuser -k ${port}/tcp 2>/dev/null || echo "No process found via fuser for port $port"
fi
sleep 1
done
# Wait for ports to be released
sleep 5
echo "=== Post-cleanup Port Check ==="
netstat -tulpn | grep -E "(9337|8085|8892|8004|9328)" || echo "All ports are now free"
# If any ports are still in use, fail fast
if netstat -tulpn | grep -E "(9337|8085|8892|8004|9328)" >/dev/null 2>&1; then
echo "❌ ERROR: Some ports are still in use after aggressive cleanup!"
echo "=== Detailed Port Analysis ==="
for port in 9337 8085 8892 8004 9328; do
echo "Port $port:"
netstat -tlnp 2>/dev/null | grep ":$port " || echo " Not in use"
lsof -i :$port 2>/dev/null || echo " No lsof info"
done
exit 1
fi
go install -tags "sqlite" -buildvcs=false
# Create clean data directory for this test run with unique timestamp and process ID
export WEED_DATA_DIR="/tmp/seaweedfs-sql-test-$(date +%s)-$$"
# Create clean data directory for this test run
export WEED_DATA_DIR="/tmp/seaweedfs-sql-test-$(date +%s)"
mkdir -p "$WEED_DATA_DIR"
chmod 777 "$WEED_DATA_DIR"
# SQLite-specific configuration
export WEED_LEVELDB2_ENABLED="false"
export WEED_SQLITE_ENABLED="true"
export WEED_SQLITE_DBFILE="$WEED_DATA_DIR/filer.db"
echo "=== SQL Store Configuration ==="
echo "Data Dir: $WEED_DATA_DIR"
echo "SQLite DB: $WEED_SQLITE_DBFILE"
echo "LEVELDB2_ENABLED: $WEED_LEVELDB2_ENABLED"
echo "SQLITE_ENABLED: $WEED_SQLITE_ENABLED"
export WEED_LEVELDB2_ENABLED="false" WEED_SQLITE_ENABLED="true" WEED_SQLITE_DBFILE="$WEED_DATA_DIR/filer.db"
set -x
weed -v 1 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9337 -volume.port=8085 -filer.port=8892 -s3.port=8004 -metricsPort=9328 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json \
> /tmp/seaweedfs-sql-server.log 2>&1 &
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
-volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
pid=$!
echo "=== Server started with PID: $pid ==="
# Wait for all SeaweedFS components to be ready
echo "Waiting for SeaweedFS components to start..."
# Check if server process is still alive before waiting
if ! kill -0 $pid 2>/dev/null; then
echo "❌ Server process died immediately after start"
echo "=== Immediate Log Check ==="
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null || echo "No log available"
exit 1
fi
sleep 5 # Give SQLite more time to initialize
for i in {1..30}; do
if curl -s http://localhost:9337/cluster/status > /dev/null 2>&1; then
echo "Master server is ready"
break
fi
echo "Waiting for master server... ($i/30)"
# Check if server process is still alive
if ! kill -0 $pid 2>/dev/null; then
echo "❌ Server process died while waiting for master"
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
exit 1
fi
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8085/status > /dev/null 2>&1; then
echo "Volume server is ready"
break
fi
echo "Waiting for volume server... ($i/30)"
if ! kill -0 $pid 2>/dev/null; then
echo "❌ Server process died while waiting for volume"
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
exit 1
fi
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8892/ > /dev/null 2>&1; then
echo "Filer (SQLite) is ready"
break
fi
echo "Waiting for filer (SQLite)... ($i/30)"
if ! kill -0 $pid 2>/dev/null; then
echo "❌ Server process died while waiting for filer"
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
exit 1
fi
sleep 2
done
# Extra wait for SQLite filer to fully initialize
echo "Giving SQLite filer extra time to initialize..."
sleep 5
for i in {1..30}; do
if curl -s http://localhost:8004/ > /dev/null 2>&1; then
echo "S3 server is ready"
break
fi
echo "Waiting for S3 server... ($i/30)"
if ! kill -0 $pid 2>/dev/null; then
echo "❌ Server process died while waiting for S3"
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
exit 1
fi
sleep 2
done
echo "All SeaweedFS components are ready!"
sleep 10
cd ../s3-tests
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
# Create and update s3tests.conf to use port 8004
cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-sql.conf
sed -i 's/port = 8000/port = 8004/g' ../docker/compose/s3tests-sql.conf
sed -i 's/:8000/:8004/g' ../docker/compose/s3tests-sql.conf
sed -i 's/localhost:8000/localhost:8004/g' ../docker/compose/s3tests-sql.conf
sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8004/g' ../docker/compose/s3tests-sql.conf
export S3TEST_CONF=../docker/compose/s3tests-sql.conf
# Debug: Show the config file contents
echo "=== S3 Config File Contents ==="
cat ../docker/compose/s3tests-sql.conf
echo "=== End Config ==="
# Additional wait for S3-Filer integration to be fully ready
echo "Waiting additional 10 seconds for S3-Filer integration..."
sleep 10
# Test S3 connection before running tests
echo "Testing S3 connection..."
# Debug: Check if SeaweedFS processes are running
echo "=== Process Status ==="
ps aux | grep -E "(weed|seaweedfs)" | grep -v grep || echo "No SeaweedFS processes found"
# Debug: Check port status
echo "=== Port Status ==="
netstat -tulpn | grep -E "(8004|9337|8085|8892)" || echo "Ports not found"
# Debug: Check server logs
echo "=== Recent Server Logs ==="
echo "--- SQL Server Log ---"
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null || echo "No SQL server log found"
echo "--- Other Logs ---"
ls -la /tmp/seaweedfs-*.log 2>/dev/null || echo "No other log files found"
for i in {1..10}; do
if curl -s -f http://localhost:8004/ > /dev/null 2>&1; then
echo "S3 connection test successful"
break
fi
echo "S3 connection test failed, retrying... ($i/10)"
# Debug: Try different HTTP methods
echo "Debug: Testing different endpoints..."
curl -s -I http://localhost:8004/ || echo "HEAD request failed"
curl -s http://localhost:8004/status || echo "Status endpoint failed"
sleep 2
done
tox -- \
s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \
s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \
-2
View File
@@ -110,5 +110,3 @@ test/s3/cors/cors.test
/test/s3/retention/filerldb2
test/s3/retention/weed-server.pid
test/s3/retention/weed-test.log
/test/s3/versioning/test-volume-data
test/s3/versioning/weed-test.log
+1 -1
View File
@@ -23,7 +23,7 @@ server: install
benchmark: install warp_install
pkill weed || true
pkill warp || true
weed server -debug=$(debug) -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json &
weed server -debug=$(debug) -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json &
warp client &
while ! nc -z localhost 8000 ; do sleep 1 ; done
warp mixed --host=127.0.0.1:8000 --access-key=some_access_key1 --secret-key=some_secret_key1 --autoterm
+2 -2
View File
@@ -10,7 +10,7 @@ services:
- 18084:18080
- 8888:8888
- 18888:18888
command: "server -ip=server1 -filer -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1"
command: "server -ip=server1 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
volumes:
- ./master-cloud.toml:/etc/seaweedfs/master.toml
depends_on:
@@ -25,4 +25,4 @@ services:
- 8889:8888
- 18889:18888
- 8334:8333
command: "server -ip=server2 -filer -s3 -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1"
command: "server -ip=server2 -filer -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
@@ -3,7 +3,7 @@ version: '3.9'
services:
server-left:
image: chrislusf/seaweedfs:local
command: "-v=0 server -ip=server-left -filer -filer.maxMB 5 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1"
command: "-v=0 server -ip=server-left -filer -filer.maxMB 5 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
volumes:
- ./s3.json:/etc/seaweedfs/s3.json
healthcheck:
@@ -13,7 +13,7 @@ services:
timeout: 30s
server-right:
image: chrislusf/seaweedfs:local
command: "-v=0 server -ip=server-right -filer -filer.maxMB 64 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1"
command: "-v=0 server -ip=server-right -filer -filer.maxMB 64 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
volumes:
- ./s3.json:/etc/seaweedfs/s3.json
healthcheck:
@@ -6,7 +6,7 @@ services:
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master -volumeSizeLimitMB=100"
command: "master -ip=master -volumeSizeLimitMB=1024"
volume:
image: chrislusf/seaweedfs:local
ports:
+1 -1
View File
@@ -6,7 +6,7 @@ services:
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master -volumeSizeLimitMB=100"
command: "master -ip=master -volumeSizeLimitMB=1024"
volume:
image: chrislusf/seaweedfs:local
ports:
+1 -1
View File
@@ -11,7 +11,7 @@ services:
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master -volumeSizeLimitMB=100"
command: "master -ip=master -volumeSizeLimitMB=1024"
volume:
image: chrislusf/seaweedfs:local
ports:
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "3.95"
appVersion: "3.94"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.0.395
version: 4.0.394
+2 -2
View File
@@ -222,13 +222,13 @@ test-with-server: start-server
test-versioning-with-configs: check-deps
@echo "Testing with different S3 configurations..."
@echo "Testing with empty folder allowed..."
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowEmptyFolder=true -filer -master.volumeSizeLimitMB=100 -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowEmptyFolder=true -filer -master.volumeSizeLimitMB=1024 -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
@sleep 5
@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
@if [ -f weed-config1.pid ]; then kill -TERM $$(cat weed-config1.pid) 2>/dev/null || true; rm -f weed-config1.pid; fi
@sleep 2
@echo "Testing with delete bucket not empty disabled..."
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=100 -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=1024 -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
@sleep 5
@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
@if [ -f weed-config2.pid ]; then kill -TERM $$(cat weed-config2.pid) 2>/dev/null || true; rm -f weed-config2.pid; fi
+2 -13
View File
@@ -164,16 +164,6 @@ func checkVersioningStatus(t *testing.T, client *s3.Client, bucketName string, e
assert.Equal(t, expectedStatus, resp.Status)
}
// checkVersioningStatusEmpty verifies that a bucket has no versioning configuration (newly created bucket)
func checkVersioningStatusEmpty(t *testing.T, client *s3.Client, bucketName string) {
resp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
// AWS S3 returns an empty versioning configuration (no Status field) for buckets that have never had versioning configured, such as newly created buckets.
assert.Empty(t, resp.Status, "Newly created bucket should have empty versioning status")
}
// putObject puts an object into a bucket
func putObject(t *testing.T, client *s3.Client, bucketName, key, content string) *s3.PutObjectOutput {
resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
@@ -294,9 +284,8 @@ func TestVersioningBasicWorkflow(t *testing.T) {
createBucket(t, client, bucketName)
defer deleteBucket(t, client, bucketName)
// Initially, versioning should be unset/empty (not suspended) for newly created buckets
// This matches AWS S3 behavior where new buckets have no versioning status
checkVersioningStatusEmpty(t, client, bucketName)
// Initially, versioning should be suspended/disabled
checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusSuspended)
// Enable versioning
enableVersioning(t, client, bucketName)
+1 -1
View File
@@ -23,7 +23,7 @@ debug_mount:
debug_server:
go build -gcflags="all=-N -l"
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1
debug_volume:
go build -tags=5BytesOffset -gcflags="all=-N -l"
+3 -2
View File
@@ -138,8 +138,9 @@ func (r *LockRing) compactSnapshots() {
r.Lock()
defer r.Unlock()
// Always attempt compaction when called, regardless of lastCompactTime
// This ensures proper cleanup even with multiple concurrent compaction requests
if r.lastCompactTime.After(r.lastUpdateTime) {
return
}
ts := time.Now()
// remove old snapshots
+38 -62
View File
@@ -238,10 +238,32 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
}
entryName, dirName := s3a.getEntryNameAndDir(input)
err = s3a.mkFile(dirName, entryName, finalParts, func(entry *filer_pb.Entry) {
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
entry.Extended[s3_constants.SeaweedFSUploadId] = []byte(*input.UploadId)
for k, v := range pentry.Extended {
if k != "key" {
entry.Extended[k] = v
}
}
if pentry.Attributes.Mime != "" {
entry.Attributes.Mime = pentry.Attributes.Mime
} else if mime != "" {
entry.Attributes.Mime = mime
}
entry.Attributes.FileSize = uint64(offset)
})
// Check if versioning is configured for this bucket BEFORE creating any files
versioningState, vErr := s3a.getVersioningState(*input.Bucket)
if vErr == nil && versioningState == s3_constants.VersioningEnabled {
if err != nil {
glog.Errorf("completeMultipartUpload %s/%s error: %v", dirName, entryName, err)
return nil, s3err.ErrInternalError
}
// Check if versioning is enabled for this bucket
versioningEnabled, vErr := s3a.isVersioningEnabled(*input.Bucket)
if vErr == nil && versioningEnabled {
// For versioned buckets, create a version and return the version ID
versionId := generateVersionId()
versionFileName := s3a.getVersionFileName(versionId)
@@ -279,8 +301,19 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
return nil, s3err.ErrInternalError
}
// For versioned buckets, don't create a main object file - all content is stored in .versions directory
// The latest version information is tracked in the .versions directory metadata
// Create a delete marker for the main object (latest version)
err = s3a.mkFile(dirName, entryName, nil, func(mainEntry *filer_pb.Entry) {
if mainEntry.Extended == nil {
mainEntry.Extended = make(map[string][]byte)
}
mainEntry.Extended[s3_constants.ExtVersionIdKey] = []byte(versionId)
mainEntry.Extended[s3_constants.ExtDeleteMarkerKey] = []byte("false") // This is the latest version, not a delete marker
})
if err != nil {
glog.Errorf("completeMultipartUpload: failed to update main entry: %v", err)
return nil, s3err.ErrInternalError
}
output = &CompleteMultipartUploadResult{
Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlEscapeObject(dirName), urlPathEscape(entryName))),
@@ -289,64 +322,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
Key: objectKey(input.Key),
VersionId: aws.String(versionId),
}
} else if vErr == nil && versioningState == s3_constants.VersioningSuspended {
// For suspended versioning, add "null" version ID metadata and return "null" version ID
err = s3a.mkFile(dirName, entryName, finalParts, func(entry *filer_pb.Entry) {
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
entry.Extended[s3_constants.ExtVersionIdKey] = []byte("null")
for k, v := range pentry.Extended {
if k != "key" {
entry.Extended[k] = v
}
}
if pentry.Attributes.Mime != "" {
entry.Attributes.Mime = pentry.Attributes.Mime
} else if mime != "" {
entry.Attributes.Mime = mime
}
entry.Attributes.FileSize = uint64(offset)
})
if err != nil {
glog.Errorf("completeMultipartUpload: failed to create suspended versioning object: %v", err)
return nil, s3err.ErrInternalError
}
// Note: Suspended versioning should NOT return VersionId field according to AWS S3 spec
output = &CompleteMultipartUploadResult{
Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlEscapeObject(dirName), urlPathEscape(entryName))),
Bucket: input.Bucket,
ETag: aws.String("\"" + filer.ETagChunks(finalParts) + "\""),
Key: objectKey(input.Key),
// VersionId field intentionally omitted for suspended versioning
}
} else {
// For non-versioned buckets, create main object file
err = s3a.mkFile(dirName, entryName, finalParts, func(entry *filer_pb.Entry) {
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
entry.Extended[s3_constants.SeaweedFSUploadId] = []byte(*input.UploadId)
for k, v := range pentry.Extended {
if k != "key" {
entry.Extended[k] = v
}
}
if pentry.Attributes.Mime != "" {
entry.Attributes.Mime = pentry.Attributes.Mime
} else if mime != "" {
entry.Attributes.Mime = mime
}
entry.Attributes.FileSize = uint64(offset)
})
if err != nil {
glog.Errorf("completeMultipartUpload %s/%s error: %v", dirName, entryName, err)
return nil, s3err.ErrInternalError
}
// For non-versioned buckets, return response without VersionId
output = &CompleteMultipartUploadResult{
Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlEscapeObject(dirName), urlPathEscape(entryName))),
+6 -39
View File
@@ -2,7 +2,6 @@ package s3api
import (
"encoding/json"
"errors"
"fmt"
"path/filepath"
"strings"
@@ -136,7 +135,7 @@ func (s3a *S3ApiServer) getBucketConfig(bucket string) (*BucketConfig, s3err.Err
// Load CORS configuration from .s3metadata
if corsConfig, err := s3a.loadCORSFromMetadata(bucket); err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
if err == filer_pb.ErrNotFound {
// Missing metadata is not an error; fall back cleanly
glog.V(2).Infof("CORS metadata not found for bucket %s, falling back to default behavior", bucket)
} else {
@@ -220,40 +219,6 @@ func (s3a *S3ApiServer) isVersioningEnabled(bucket string) (bool, error) {
return config.Versioning == s3_constants.VersioningEnabled || config.ObjectLockConfig != nil, nil
}
// isVersioningConfigured checks if versioning has been configured (either Enabled or Suspended)
func (s3a *S3ApiServer) isVersioningConfigured(bucket string) (bool, error) {
config, errCode := s3a.getBucketConfig(bucket)
if errCode != s3err.ErrNone {
if errCode == s3err.ErrNoSuchBucket {
return false, filer_pb.ErrNotFound
}
return false, fmt.Errorf("failed to get bucket config: %v", errCode)
}
// Versioning is configured if explicitly set to either "Enabled" or "Suspended"
// OR if object lock is enabled (which forces versioning)
return config.Versioning != "" || config.ObjectLockConfig != nil, nil
}
// getVersioningState returns the detailed versioning state for a bucket
func (s3a *S3ApiServer) getVersioningState(bucket string) (string, error) {
config, errCode := s3a.getBucketConfig(bucket)
if errCode != s3err.ErrNone {
if errCode == s3err.ErrNoSuchBucket {
return "", filer_pb.ErrNotFound
}
return "", fmt.Errorf("failed to get bucket config: %v", errCode)
}
// If object lock is enabled, versioning must be enabled regardless of explicit setting
if config.ObjectLockConfig != nil {
return s3_constants.VersioningEnabled, nil
}
// Return the explicit versioning status (empty string means never configured)
return config.Versioning, nil
}
// getBucketVersioningStatus returns the versioning status for a bucket
func (s3a *S3ApiServer) getBucketVersioningStatus(bucket string) (string, s3err.ErrorCode) {
config, errCode := s3a.getBucketConfig(bucket)
@@ -261,8 +226,10 @@ func (s3a *S3ApiServer) getBucketVersioningStatus(bucket string) (string, s3err.
return "", errCode
}
// Return exactly what's stored - empty string means versioning was never configured
// This matches AWS S3 behavior where new buckets have no Status field in GetBucketVersioning response
if config.Versioning == "" {
return s3_constants.VersioningSuspended, s3err.ErrNone
}
return config.Versioning, s3err.ErrNone
}
@@ -311,7 +278,7 @@ func (s3a *S3ApiServer) loadCORSFromMetadata(bucket string) (*cors.CORSConfigura
entry, err := s3a.getEntry("", bucketMetadataPath)
if err != nil {
glog.V(3).Infof("loadCORSFromMetadata: error retrieving metadata for bucket %s: %v", bucket, err)
return nil, fmt.Errorf("error retrieving CORS metadata for bucket %s: %w", bucket, err)
return nil, fmt.Errorf("error retrieving metadata for bucket %s: %v", bucket, err)
}
if entry == nil {
glog.V(3).Infof("loadCORSFromMetadata: no metadata entry found for bucket %s", bucket)
+5 -16
View File
@@ -714,22 +714,11 @@ func (s3a *S3ApiServer) GetBucketVersioningHandler(w http.ResponseWriter, r *htt
return
}
// AWS S3 behavior: If versioning was never configured, don't return Status field
var response *s3.PutBucketVersioningInput
if versioningStatus == "" {
// No versioning configuration - return empty response (no Status field)
response = &s3.PutBucketVersioningInput{
VersioningConfiguration: &s3.VersioningConfiguration{},
}
} else {
// Versioning was explicitly configured - return the status
response = &s3.PutBucketVersioningInput{
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String(versioningStatus),
},
}
}
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, response)
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, &s3.PutBucketVersioningInput{
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String(versioningStatus),
},
})
}
// PutBucketVersioningHandler Put bucket Versioning
+16 -40
View File
@@ -136,8 +136,8 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
// Check for specific version ID in query parameters
versionId := r.URL.Query().Get("versionId")
// Check if versioning is configured for the bucket (Enabled or Suspended)
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
// Check if versioning is enabled for the bucket
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
@@ -148,11 +148,9 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
return
}
glog.V(1).Infof("GetObject: bucket %s, object %s, versioningConfigured=%v, versionId=%s", bucket, object, versioningConfigured, versionId)
var destUrl string
if versioningConfigured {
if versioningEnabled {
// Handle versioned GET - all versions are stored in .versions directory
var targetVersionId string
var entry *filer_pb.Entry
@@ -169,10 +167,10 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
targetVersionId = versionId
} else {
// Request for latest version
glog.V(1).Infof("GetObject: requesting latest version for %s/%s", bucket, object)
glog.V(2).Infof("GetObject: requesting latest version for %s/%s", bucket, object)
entry, err = s3a.getLatestObjectVersion(bucket, object)
if err != nil {
glog.Errorf("GetObject: Failed to get latest version for %s/%s: %v", bucket, object, err)
glog.Errorf("Failed to get latest version: %v", err)
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
return
}
@@ -181,10 +179,6 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
targetVersionId = string(versionIdBytes)
}
}
// If no version ID found in entry, this is a pre-versioning object
if targetVersionId == "" {
targetVersionId = "null"
}
}
// Check if this is a delete marker
@@ -195,17 +189,10 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
}
}
// Determine the actual file path based on whether this is a versioned or pre-versioning object
if targetVersionId == "null" {
// Pre-versioning object - stored as regular file
destUrl = s3a.toFilerUrl(bucket, object)
glog.V(2).Infof("GetObject: pre-versioning object URL: %s", destUrl)
} else {
// Versioned object - stored in .versions directory
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
glog.V(2).Infof("GetObject: version %s URL: %s", targetVersionId, destUrl)
}
// All versions are stored in .versions directory
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
glog.V(2).Infof("GetObject: version %s URL: %s", targetVersionId, destUrl)
// Set version ID in response header
w.Header().Set("x-amz-version-id", targetVersionId)
@@ -228,8 +215,8 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
// Check for specific version ID in query parameters
versionId := r.URL.Query().Get("versionId")
// Check if versioning is configured for the bucket (Enabled or Suspended)
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
// Check if versioning is enabled for the bucket
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
@@ -242,7 +229,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
var destUrl string
if versioningConfigured {
if versioningEnabled {
// Handle versioned HEAD - all versions are stored in .versions directory
var targetVersionId string
var entry *filer_pb.Entry
@@ -271,10 +258,6 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
targetVersionId = string(versionIdBytes)
}
}
// If no version ID found in entry, this is a pre-versioning object
if targetVersionId == "" {
targetVersionId = "null"
}
}
// Check if this is a delete marker
@@ -285,17 +268,10 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
}
}
// Determine the actual file path based on whether this is a versioned or pre-versioning object
if targetVersionId == "null" {
// Pre-versioning object - stored as regular file
destUrl = s3a.toFilerUrl(bucket, object)
glog.V(2).Infof("HeadObject: pre-versioning object URL: %s", destUrl)
} else {
// Versioned object - stored in .versions directory
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
glog.V(2).Infof("HeadObject: version %s URL: %s", targetVersionId, destUrl)
}
// All versions are stored in .versions directory
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
glog.V(2).Infof("HeadObject: version %s URL: %s", targetVersionId, destUrl)
// Set version ID in response header
w.Header().Set("x-amz-version-id", targetVersionId)
+39 -178
View File
@@ -38,9 +38,9 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
cpSrcPath = r.Header.Get("X-Amz-Copy-Source")
}
srcBucket, srcObject, srcVersionId := pathToBucketObjectAndVersion(cpSrcPath)
srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
glog.V(3).Infof("CopyObjectHandler %s %s (version: %s) => %s %s", srcBucket, srcObject, srcVersionId, dstBucket, dstObject)
glog.V(3).Infof("CopyObjectHandler %s %s => %s %s", srcBucket, srcObject, dstBucket, dstObject)
replaceMeta, replaceTagging := replaceDirective(r.Header)
@@ -76,41 +76,9 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
return
}
// Get detailed versioning state for source bucket
srcVersioningState, err := s3a.getVersioningState(srcBucket)
if err != nil {
glog.Errorf("Error checking versioning state for source bucket %s: %v", srcBucket, err)
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
return
}
// Get the source entry with version awareness based on versioning state
var entry *filer_pb.Entry
if srcVersionId != "" {
// Specific version requested - always use version-aware retrieval
entry, err = s3a.getSpecificObjectVersion(srcBucket, srcObject, srcVersionId)
} else if srcVersioningState == s3_constants.VersioningEnabled {
// Versioning enabled - get latest version from .versions directory
entry, err = s3a.getLatestObjectVersion(srcBucket, srcObject)
} else if srcVersioningState == s3_constants.VersioningSuspended {
// Versioning suspended - current object is stored as regular file ("null" version)
// Try regular file first, fall back to latest version if needed
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
dir, name := srcPath.DirAndName()
entry, err = s3a.getEntry(dir, name)
if err != nil {
// If regular file doesn't exist, try latest version as fallback
glog.V(2).Infof("CopyObject: regular file not found for suspended versioning, trying latest version")
entry, err = s3a.getLatestObjectVersion(srcBucket, srcObject)
}
} else {
// No versioning configured - use regular retrieval
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
dir, name := srcPath.DirAndName()
entry, err = s3a.getEntry(dir, name)
}
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
dir, name := srcPath.DirAndName()
entry, err := s3a.getEntry(dir, name)
if err != nil || entry.IsDirectory {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
return
@@ -170,108 +138,43 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
dstEntry.Chunks = dstChunks
}
// Check if destination bucket has versioning configured
dstVersioningConfigured, err := s3a.isVersioningConfigured(dstBucket)
if err != nil {
glog.Errorf("Error checking versioning status for destination bucket %s: %v", dstBucket, err)
// Save the new entry
dstPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject))
dstDir, dstName := dstPath.DirAndName()
// Check if destination exists and remove it first (S3 copy overwrites)
if exists, _ := s3a.exists(dstDir, dstName, false); exists {
if err := s3a.rm(dstDir, dstName, false, false); err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
}
// Create the new file
if err := s3a.mkFile(dstDir, dstName, dstEntry.Chunks, func(entry *filer_pb.Entry) {
entry.Attributes = dstEntry.Attributes
entry.Extended = dstEntry.Extended
}); err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
var dstVersionId string
var etag string
if dstVersioningConfigured {
// For versioned destination, create a new version
dstVersionId = generateVersionId()
glog.V(2).Infof("CopyObjectHandler: creating version %s for destination %s/%s", dstVersionId, dstBucket, dstObject)
// Add version metadata to the entry
if dstEntry.Extended == nil {
dstEntry.Extended = make(map[string][]byte)
}
dstEntry.Extended[s3_constants.ExtVersionIdKey] = []byte(dstVersionId)
// Calculate ETag for versioning
filerEntry := &filer.Entry{
FullPath: util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject)),
Attr: filer.Attr{
FileSize: dstEntry.Attributes.FileSize,
Mtime: time.Unix(dstEntry.Attributes.Mtime, 0),
Crtime: time.Unix(dstEntry.Attributes.Crtime, 0),
Mime: dstEntry.Attributes.Mime,
},
Chunks: dstEntry.Chunks,
}
etag = filer.ETagEntry(filerEntry)
if !strings.HasPrefix(etag, "\"") {
etag = "\"" + etag + "\""
}
dstEntry.Extended[s3_constants.ExtETagKey] = []byte(etag)
// Create version file
versionFileName := s3a.getVersionFileName(dstVersionId)
versionObjectPath := dstObject + ".versions/" + versionFileName
bucketDir := s3a.option.BucketsPath + "/" + dstBucket
if err := s3a.mkFile(bucketDir, versionObjectPath, dstEntry.Chunks, func(entry *filer_pb.Entry) {
entry.Attributes = dstEntry.Attributes
entry.Extended = dstEntry.Extended
}); err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
// Update the .versions directory metadata
err = s3a.updateLatestVersionInDirectory(dstBucket, dstObject, dstVersionId, versionFileName)
if err != nil {
glog.Errorf("CopyObjectHandler: failed to update latest version in directory: %v", err)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
// Set version ID in response header
w.Header().Set("x-amz-version-id", dstVersionId)
} else {
// For non-versioned destination, use regular copy
dstPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject))
dstDir, dstName := dstPath.DirAndName()
// Check if destination exists and remove it first (S3 copy overwrites)
if exists, _ := s3a.exists(dstDir, dstName, false); exists {
if err := s3a.rm(dstDir, dstName, false, false); err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
}
// Create the new file
if err := s3a.mkFile(dstDir, dstName, dstEntry.Chunks, func(entry *filer_pb.Entry) {
entry.Attributes = dstEntry.Attributes
entry.Extended = dstEntry.Extended
}); err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
// Calculate ETag
filerEntry := &filer.Entry{
FullPath: dstPath,
Attr: filer.Attr{
FileSize: dstEntry.Attributes.FileSize,
Mtime: time.Unix(dstEntry.Attributes.Mtime, 0),
Crtime: time.Unix(dstEntry.Attributes.Crtime, 0),
Mime: dstEntry.Attributes.Mime,
},
Chunks: dstEntry.Chunks,
}
etag = filer.ETagEntry(filerEntry)
// Convert filer_pb.Entry to filer.Entry for ETag calculation
filerEntry := &filer.Entry{
FullPath: dstPath,
Attr: filer.Attr{
FileSize: dstEntry.Attributes.FileSize,
Mtime: time.Unix(dstEntry.Attributes.Mtime, 0),
Crtime: time.Unix(dstEntry.Attributes.Crtime, 0),
Mime: dstEntry.Attributes.Mime,
},
Chunks: dstEntry.Chunks,
}
setEtag(w, etag)
setEtag(w, filer.ETagEntry(filerEntry))
response := CopyObjectResult{
ETag: etag,
ETag: filer.ETagEntry(filerEntry),
LastModified: time.Now().UTC(),
}
@@ -288,18 +191,6 @@ func pathToBucketAndObject(path string) (bucket, object string) {
return parts[0], "/"
}
func pathToBucketObjectAndVersion(path string) (bucket, object, versionId string) {
// Parse versionId from query string if present
// Format: /bucket/object?versionId=version-id
if idx := strings.Index(path, "?versionId="); idx != -1 {
versionId = path[idx+len("?versionId="):] // dynamically calculate length
path = path[:idx]
}
bucket, object = pathToBucketAndObject(path)
return bucket, object, versionId
}
type CopyPartResult struct {
LastModified time.Time `xml:"LastModified"`
ETag string `xml:"ETag"`
@@ -317,7 +208,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
cpSrcPath = r.Header.Get("X-Amz-Copy-Source")
}
srcBucket, srcObject, srcVersionId := pathToBucketObjectAndVersion(cpSrcPath)
srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
// If source object is empty or bucket is empty, reply back invalid copy source.
if srcObject == "" || srcBucket == "" {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
@@ -348,40 +239,10 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
return
}
// Get detailed versioning state for source bucket
srcVersioningState, err := s3a.getVersioningState(srcBucket)
if err != nil {
glog.Errorf("Error checking versioning state for source bucket %s: %v", srcBucket, err)
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
return
}
// Get the source entry with version awareness based on versioning state
var entry *filer_pb.Entry
if srcVersionId != "" {
// Specific version requested - always use version-aware retrieval
entry, err = s3a.getSpecificObjectVersion(srcBucket, srcObject, srcVersionId)
} else if srcVersioningState == s3_constants.VersioningEnabled {
// Versioning enabled - get latest version from .versions directory
entry, err = s3a.getLatestObjectVersion(srcBucket, srcObject)
} else if srcVersioningState == s3_constants.VersioningSuspended {
// Versioning suspended - current object is stored as regular file ("null" version)
// Try regular file first, fall back to latest version if needed
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
dir, name := srcPath.DirAndName()
entry, err = s3a.getEntry(dir, name)
if err != nil {
// If regular file doesn't exist, try latest version as fallback
glog.V(2).Infof("CopyObjectPart: regular file not found for suspended versioning, trying latest version")
entry, err = s3a.getLatestObjectVersion(srcBucket, srcObject)
}
} else {
// No versioning configured - use regular retrieval
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
dir, name := srcPath.DirAndName()
entry, err = s3a.getEntry(dir, name)
}
// Get source entry
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
dir, name := srcPath.DirAndName()
entry, err := s3a.getEntry(dir, name)
if err != nil || entry.IsDirectory {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
return
+30 -91
View File
@@ -32,8 +32,8 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
// Check for specific version ID in query parameters
versionId := r.URL.Query().Get("versionId")
// Check if versioning is configured for the bucket (Enabled or Suspended)
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
// Check if versioning is enabled for the bucket
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
@@ -49,7 +49,7 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
}
if versioningConfigured {
if versioningEnabled {
// Handle versioned delete
if versionId != "" {
// Check object lock permissions before deleting specific version
@@ -137,10 +137,8 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
// ObjectIdentifier represents an object to be deleted with its key name and optional version ID.
type ObjectIdentifier struct {
Key string `xml:"Key"`
VersionId string `xml:"VersionId,omitempty"`
DeleteMarker bool `xml:"DeleteMarker,omitempty"`
DeleteMarkerVersionId string `xml:"DeleteMarkerVersionId,omitempty"`
Key string `xml:"Key"`
VersionId string `xml:"VersionId,omitempty"`
}
// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
@@ -203,8 +201,8 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
}
// Check if versioning is configured for the bucket (needed for object lock checks)
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
// Check if versioning is enabled for the bucket (needed for object lock checks)
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
@@ -224,7 +222,7 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
}
// Check object lock permissions before deletion (only for versioned buckets)
if versioningConfigured {
if versioningEnabled {
// Validate governance bypass for this specific object
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object.Key)
if err := s3a.enforceObjectLockProtections(r, bucket, object.Key, object.VersionId, governanceBypassAllowed); err != nil {
@@ -238,90 +236,31 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
continue
}
}
var deleteVersionId string
var isDeleteMarker bool
if versioningConfigured {
// Handle versioned delete
if object.VersionId != "" {
// Delete specific version
err := s3a.deleteSpecificObjectVersion(bucket, object.Key, object.VersionId)
if err != nil {
deleteErrors = append(deleteErrors, DeleteError{
Code: "",
Message: err.Error(),
Key: object.Key,
VersionId: object.VersionId,
})
continue
}
deleteVersionId = object.VersionId
} else {
// Create delete marker (logical delete)
deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object.Key)
if err != nil {
deleteErrors = append(deleteErrors, DeleteError{
Code: "",
Message: err.Error(),
Key: object.Key,
VersionId: object.VersionId,
})
continue
}
deleteVersionId = deleteMarkerVersionId
isDeleteMarker = true
}
// Add to successful deletions with version info
deletedObject := ObjectIdentifier{
Key: object.Key,
VersionId: deleteVersionId,
DeleteMarker: isDeleteMarker,
}
// For delete markers, also set DeleteMarkerVersionId field
if isDeleteMarker {
deletedObject.DeleteMarkerVersionId = deleteVersionId
// Don't set VersionId for delete markers, use DeleteMarkerVersionId instead
deletedObject.VersionId = ""
}
if !deleteObjects.Quiet {
deletedObjects = append(deletedObjects, deletedObject)
}
if isDeleteMarker {
// For delete markers, we don't need to track directories for cleanup
continue
}
} else {
// Handle non-versioned delete (original logic)
lastSeparator := strings.LastIndex(object.Key, "/")
parentDirectoryPath, entryName, isDeleteData, isRecursive := "", object.Key, true, false
if lastSeparator > 0 && lastSeparator+1 < len(object.Key) {
entryName = object.Key[lastSeparator+1:]
parentDirectoryPath = "/" + object.Key[:lastSeparator]
}
parentDirectoryPath = fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, parentDirectoryPath)
err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
if err == nil {
directoriesWithDeletion[parentDirectoryPath]++
deletedObjects = append(deletedObjects, object)
} else if strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) {
deletedObjects = append(deletedObjects, object)
} else {
delete(directoriesWithDeletion, parentDirectoryPath)
deleteErrors = append(deleteErrors, DeleteError{
Code: "",
Message: err.Error(),
Key: object.Key,
VersionId: object.VersionId,
})
}
lastSeparator := strings.LastIndex(object.Key, "/")
parentDirectoryPath, entryName, isDeleteData, isRecursive := "", object.Key, true, false
if lastSeparator > 0 && lastSeparator+1 < len(object.Key) {
entryName = object.Key[lastSeparator+1:]
parentDirectoryPath = "/" + object.Key[:lastSeparator]
}
parentDirectoryPath = fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, parentDirectoryPath)
err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
if err == nil {
directoriesWithDeletion[parentDirectoryPath]++
deletedObjects = append(deletedObjects, object)
} else if strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) {
deletedObjects = append(deletedObjects, object)
} else {
delete(directoriesWithDeletion, parentDirectoryPath)
deleteErrors = append(deleteErrors, DeleteError{
Code: "",
Message: err.Error(),
Key: object.Key,
VersionId: object.VersionId,
})
}
if auditLog != nil {
auditLog.Key = object.Key
auditLog.Key = entryName
s3err.PostAccessLog(*auditLog)
}
}
+7 -152
View File
@@ -95,8 +95,8 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
return
}
} else {
// Get detailed versioning state for the bucket
versioningState, err := s3a.getVersioningState(bucket)
// Check if versioning is enabled for the bucket
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
@@ -107,10 +107,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
return
}
versioningEnabled := (versioningState == s3_constants.VersioningEnabled)
versioningConfigured := (versioningState != "")
glog.V(1).Infof("PutObjectHandler: bucket %s, object %s, versioningState=%s", bucket, object, versioningState)
glog.V(1).Infof("PutObjectHandler: bucket %s, object %s, versioningEnabled=%v", bucket, object, versioningEnabled)
// Validate object lock headers before processing
if err := s3a.validateObjectLockHeaders(r, versioningEnabled); err != nil {
@@ -121,7 +118,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
// For non-versioned buckets, check if existing object has object lock protections
// that would prevent overwrite (PUT operations overwrite existing objects in non-versioned buckets)
if !versioningConfigured {
if !versioningEnabled {
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
if err := s3a.enforceObjectLockProtections(r, bucket, object, "", governanceBypassAllowed); err != nil {
glog.V(2).Infof("PutObjectHandler: object lock permissions check failed for %s/%s: %v", bucket, object, err)
@@ -130,8 +127,8 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
}
}
if versioningState == s3_constants.VersioningEnabled {
// Handle enabled versioning - create new versions with real version IDs
if versioningEnabled {
// Handle versioned PUT
glog.V(1).Infof("PutObjectHandler: using versioned PUT for %s/%s", bucket, object)
versionId, etag, errCode := s3a.putVersionedObject(r, bucket, object, dataReader, objectContentType)
if errCode != s3err.ErrNone {
@@ -144,24 +141,10 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
w.Header().Set("x-amz-version-id", versionId)
}
// Set ETag in response
setEtag(w, etag)
} else if versioningState == s3_constants.VersioningSuspended {
// Handle suspended versioning - overwrite with "null" version ID but preserve existing versions
glog.V(1).Infof("PutObjectHandler: using suspended versioning PUT for %s/%s", bucket, object)
etag, errCode := s3a.putSuspendedVersioningObject(r, bucket, object, dataReader, objectContentType)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
// Note: Suspended versioning should NOT return x-amz-version-id header according to AWS S3 spec
// The object is stored with "null" version internally but no version header is returned
// Set ETag in response
setEtag(w, etag)
} else {
// Handle regular PUT (never configured versioning)
// Handle regular PUT (non-versioned)
glog.V(1).Infof("PutObjectHandler: using regular PUT for %s/%s", bucket, object)
uploadUrl := s3a.toFilerUrl(bucket, object)
if objectContentType == "" {
@@ -175,7 +158,6 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
return
}
// No version ID header for never-configured versioning
setEtag(w, etag)
}
}
@@ -292,133 +274,6 @@ func (s3a *S3ApiServer) maybeGetFilerJwtAuthorizationToken(isWrite bool) string
// putVersionedObject handles PUT operations for versioned buckets using the new layout
// where all versions (including latest) are stored in the .versions directory
func (s3a *S3ApiServer) putSuspendedVersioningObject(r *http.Request, bucket, object string, dataReader io.Reader, objectContentType string) (etag string, errCode s3err.ErrorCode) {
// For suspended versioning, store as regular object (version ID "null") but preserve existing versions
glog.V(2).Infof("putSuspendedVersioningObject: creating null version for %s/%s", bucket, object)
uploadUrl := s3a.toFilerUrl(bucket, object)
if objectContentType == "" {
dataReader = mimeDetect(r, dataReader)
}
etag, errCode = s3a.putToFiler(r, uploadUrl, dataReader, "", bucket)
if errCode != s3err.ErrNone {
glog.Errorf("putSuspendedVersioningObject: failed to upload object: %v", errCode)
return "", errCode
}
// Get the uploaded entry to add version metadata indicating this is "null" version
bucketDir := s3a.option.BucketsPath + "/" + bucket
entry, err := s3a.getEntry(bucketDir, object)
if err != nil {
glog.Errorf("putSuspendedVersioningObject: failed to get object entry: %v", err)
return "", s3err.ErrInternalError
}
// Add metadata to indicate this is a "null" version for suspended versioning
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
entry.Extended[s3_constants.ExtVersionIdKey] = []byte("null")
// Extract and store object lock metadata from request headers (if any)
if err := s3a.extractObjectLockMetadataFromRequest(r, entry); err != nil {
glog.Errorf("putSuspendedVersioningObject: failed to extract object lock metadata: %v", err)
return "", s3err.ErrInvalidRequest
}
// Update the entry with metadata
err = s3a.mkFile(bucketDir, object, entry.Chunks, func(updatedEntry *filer_pb.Entry) {
updatedEntry.Extended = entry.Extended
updatedEntry.Attributes = entry.Attributes
updatedEntry.Chunks = entry.Chunks
})
if err != nil {
glog.Errorf("putSuspendedVersioningObject: failed to update object metadata: %v", err)
return "", s3err.ErrInternalError
}
// Update all existing versions/delete markers to set IsLatest=false since "null" is now latest
err = s3a.updateIsLatestFlagsForSuspendedVersioning(bucket, object)
if err != nil {
glog.Warningf("putSuspendedVersioningObject: failed to update IsLatest flags: %v", err)
// Don't fail the request, but log the warning
}
glog.V(2).Infof("putSuspendedVersioningObject: successfully created null version for %s/%s", bucket, object)
return etag, s3err.ErrNone
}
// updateIsLatestFlagsForSuspendedVersioning sets IsLatest=false on all existing versions/delete markers
// when a new "null" version becomes the latest during suspended versioning
func (s3a *S3ApiServer) updateIsLatestFlagsForSuspendedVersioning(bucket, object string) error {
bucketDir := s3a.option.BucketsPath + "/" + bucket
cleanObject := strings.TrimPrefix(object, "/")
versionsObjectPath := cleanObject + ".versions"
versionsDir := bucketDir + "/" + versionsObjectPath
glog.V(2).Infof("updateIsLatestFlagsForSuspendedVersioning: updating flags for %s/%s", bucket, cleanObject)
// Check if .versions directory exists
_, err := s3a.getEntry(bucketDir, versionsObjectPath)
if err != nil {
// No .versions directory exists, nothing to update
glog.V(2).Infof("updateIsLatestFlagsForSuspendedVersioning: no .versions directory for %s/%s", bucket, cleanObject)
return nil
}
// List all entries in .versions directory
entries, _, err := s3a.list(versionsDir, "", "", false, 1000)
if err != nil {
return fmt.Errorf("failed to list versions directory: %v", err)
}
glog.V(2).Infof("updateIsLatestFlagsForSuspendedVersioning: found %d entries to update", len(entries))
// Update each version/delete marker to set IsLatest=false
for _, entry := range entries {
if entry.Extended == nil {
continue
}
// Check if this entry has a version ID (it should be a version or delete marker)
versionIdBytes, hasVersionId := entry.Extended[s3_constants.ExtVersionIdKey]
if !hasVersionId {
continue
}
versionId := string(versionIdBytes)
glog.V(2).Infof("updateIsLatestFlagsForSuspendedVersioning: setting IsLatest=false for version %s", versionId)
// Update the entry to set IsLatest=false (we don't explicitly store this flag,
// it's determined by comparison with latest version metadata)
// We need to clear the latest version metadata from the .versions directory
// so that our getObjectVersionList function will correctly show IsLatest=false
}
// Clear the latest version metadata from .versions directory since "null" is now latest
versionsEntry, err := s3a.getEntry(bucketDir, versionsObjectPath)
if err == nil && versionsEntry.Extended != nil {
// Remove latest version metadata so all versions show IsLatest=false
delete(versionsEntry.Extended, s3_constants.ExtLatestVersionIdKey)
delete(versionsEntry.Extended, s3_constants.ExtLatestVersionFileNameKey)
// Update the .versions directory entry
err = s3a.mkFile(bucketDir, versionsObjectPath, versionsEntry.Chunks, func(updatedEntry *filer_pb.Entry) {
updatedEntry.Extended = versionsEntry.Extended
updatedEntry.Attributes = versionsEntry.Attributes
updatedEntry.Chunks = versionsEntry.Chunks
})
if err != nil {
return fmt.Errorf("failed to update .versions directory metadata: %v", err)
}
glog.V(2).Infof("updateIsLatestFlagsForSuspendedVersioning: cleared latest version metadata for %s/%s", bucket, cleanObject)
}
return nil
}
func (s3a *S3ApiServer) putVersionedObject(r *http.Request, bucket, object string, dataReader io.Reader, objectContentType string) (versionId string, etag string, errCode s3err.ErrorCode) {
// Generate version ID
versionId = generateVersionId()
+72 -350
View File
@@ -2,6 +2,7 @@ package s3api
import (
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/xml"
"fmt"
@@ -47,26 +48,20 @@ type ListObjectVersionsResult struct {
CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"`
}
// generateVersionId creates a unique version ID that preserves chronological order
// generateVersionId creates a unique version ID
func generateVersionId() string {
// Use nanosecond timestamp to ensure chronological ordering
// Format as 16-digit hex (first 16 chars of version ID)
now := time.Now().UnixNano()
timestampHex := fmt.Sprintf("%016x", now)
// Generate random 8 bytes for uniqueness (last 16 chars of version ID)
randBytes := make([]byte, 8)
// Generate a random 16-byte value
randBytes := make([]byte, 16)
if _, err := rand.Read(randBytes); err != nil {
glog.Errorf("Failed to generate random bytes for version ID: %v", err)
// Fallback to timestamp-only if random generation fails
return timestampHex + "0000000000000000"
return ""
}
// Combine timestamp (16 chars) + random (16 chars) = 32 chars total
randomHex := hex.EncodeToString(randBytes)
versionId := timestampHex + randomHex
// Hash with current timestamp for uniqueness
hash := sha256.Sum256(append(randBytes, []byte(fmt.Sprintf("%d", time.Now().UnixNano()))...))
return versionId
// Return first 32 characters of hex string (same length as AWS S3 version IDs)
return hex.EncodeToString(hash[:])[:32]
}
// getVersionedObjectDir returns the directory path for storing object versions
@@ -127,20 +122,59 @@ func (s3a *S3ApiServer) createDeleteMarker(bucket, object string) (string, error
func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdMarker, delimiter string, maxKeys int) (*ListObjectVersionsResult, error) {
var allVersions []interface{} // Can contain VersionEntry or DeleteMarkerEntry
// Track objects that have been processed to avoid duplicates
processedObjects := make(map[string]bool)
// Track version IDs globally to prevent duplicates throughout the listing
seenVersionIds := make(map[string]bool)
// Recursively find all .versions directories in the bucket
bucketPath := path.Join(s3a.option.BucketsPath, bucket)
err := s3a.findVersionsRecursively(bucketPath, "", &allVersions, processedObjects, seenVersionIds, bucket, prefix)
// List all entries in bucket
entries, _, err := s3a.list(path.Join(s3a.option.BucketsPath, bucket), prefix, keyMarker, false, uint32(maxKeys*2))
if err != nil {
return nil, err
}
// Sort by key, then by LastModified (newest first), then by VersionId for deterministic ordering
// For each entry, check if it's a .versions directory
for _, entry := range entries {
if !entry.IsDirectory {
continue
}
// Check if this is a .versions directory
if !strings.HasSuffix(entry.Name, ".versions") {
continue
}
// Extract object name from .versions directory name
objectKey := strings.TrimSuffix(entry.Name, ".versions")
versions, err := s3a.getObjectVersionList(bucket, objectKey)
if err != nil {
glog.Warningf("Failed to get versions for object %s: %v", objectKey, err)
continue
}
for _, version := range versions {
if version.IsDeleteMarker {
deleteMarker := &DeleteMarkerEntry{
Key: objectKey,
VersionId: version.VersionId,
IsLatest: version.IsLatest,
LastModified: version.LastModified,
Owner: CanonicalUser{ID: "unknown", DisplayName: "unknown"},
}
allVersions = append(allVersions, deleteMarker)
} else {
versionEntry := &VersionEntry{
Key: objectKey,
VersionId: version.VersionId,
IsLatest: version.IsLatest,
LastModified: version.LastModified,
ETag: version.ETag,
Size: version.Size,
Owner: CanonicalUser{ID: "unknown", DisplayName: "unknown"},
StorageClass: "STANDARD",
}
allVersions = append(allVersions, versionEntry)
}
}
}
// Sort by key, then by LastModified and VersionId
sort.Slice(allVersions, func(i, j int) bool {
var keyI, keyJ string
var lastModifiedI, lastModifiedJ time.Time
@@ -168,20 +202,13 @@ func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdM
versionIdJ = v.VersionId
}
// First sort by object key
if keyI != keyJ {
return keyI < keyJ
}
// Then by modification time (newest first) - but use nanosecond precision for ties
timeDiff := lastModifiedI.Sub(lastModifiedJ)
if timeDiff.Abs() > time.Millisecond {
if !lastModifiedI.Equal(lastModifiedJ) {
return lastModifiedI.After(lastModifiedJ)
}
// For very close timestamps (within 1ms), use version ID for deterministic ordering
// Sort version IDs in reverse lexicographic order to maintain newest-first semantics
return versionIdI > versionIdJ
return versionIdI < versionIdJ
})
// Build result
@@ -210,10 +237,6 @@ func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdM
}
}
// Always initialize empty slices so boto3 gets the expected fields even when empty
result.Versions = make([]VersionEntry, 0)
result.DeleteMarkers = make([]DeleteMarkerEntry, 0)
// Add versions to result
for _, version := range allVersions {
switch v := version.(type) {
@@ -227,128 +250,6 @@ func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdM
return result, nil
}
// findVersionsRecursively searches for all .versions directories and regular files recursively
func (s3a *S3ApiServer) findVersionsRecursively(currentPath, relativePath string, allVersions *[]interface{}, processedObjects map[string]bool, seenVersionIds map[string]bool, bucket, prefix string) error {
// List entries in current directory
entries, _, err := s3a.list(currentPath, "", "", false, 1000)
if err != nil {
return err
}
for _, entry := range entries {
entryPath := path.Join(relativePath, entry.Name)
// Skip if this doesn't match the prefix filter
if prefix != "" && !strings.HasPrefix(entryPath, strings.TrimPrefix(prefix, "/")) {
continue
}
if entry.IsDirectory {
// Skip .uploads directory (multipart upload temporary files)
if strings.HasPrefix(entry.Name, ".uploads") {
continue
}
// Check if this is a .versions directory
if strings.HasSuffix(entry.Name, ".versions") {
// Extract object name from .versions directory name
objectKey := strings.TrimSuffix(entryPath, ".versions")
processedObjects[objectKey] = true
glog.V(2).Infof("findVersionsRecursively: found .versions directory for object %s", objectKey)
versions, err := s3a.getObjectVersionList(bucket, objectKey)
if err != nil {
glog.Warningf("Failed to get versions for object %s: %v", objectKey, err)
continue
}
for _, version := range versions {
// Check for duplicate version IDs and skip if already seen
versionKey := objectKey + ":" + version.VersionId
if seenVersionIds[versionKey] {
glog.Warningf("findVersionsRecursively: duplicate version %s for object %s detected, skipping", version.VersionId, objectKey)
continue
}
seenVersionIds[versionKey] = true
if version.IsDeleteMarker {
deleteMarker := &DeleteMarkerEntry{
Key: objectKey,
VersionId: version.VersionId,
IsLatest: version.IsLatest,
LastModified: version.LastModified,
Owner: CanonicalUser{ID: "unknown", DisplayName: "unknown"},
}
*allVersions = append(*allVersions, deleteMarker)
} else {
versionEntry := &VersionEntry{
Key: objectKey,
VersionId: version.VersionId,
IsLatest: version.IsLatest,
LastModified: version.LastModified,
ETag: version.ETag,
Size: version.Size,
Owner: CanonicalUser{ID: "unknown", DisplayName: "unknown"},
StorageClass: "STANDARD",
}
*allVersions = append(*allVersions, versionEntry)
}
}
} else {
// Recursively search subdirectories
fullPath := path.Join(currentPath, entry.Name)
err := s3a.findVersionsRecursively(fullPath, entryPath, allVersions, processedObjects, seenVersionIds, bucket, prefix)
if err != nil {
glog.Warningf("Error searching subdirectory %s: %v", entryPath, err)
continue
}
}
} else {
// This is a regular file - check if it's a pre-versioning object
objectKey := entryPath
// Skip if this object already has a .versions directory (already processed)
if processedObjects[objectKey] {
continue
}
// This is a pre-versioning object - treat it as a version with VersionId="null"
glog.V(2).Infof("findVersionsRecursively: found pre-versioning object %s", objectKey)
// Check if this null version should be marked as latest
// It's only latest if there's no .versions directory OR no latest version metadata
isLatest := true
versionsObjectPath := objectKey + ".versions"
if versionsEntry, err := s3a.getEntry(currentPath, versionsObjectPath); err == nil {
// .versions directory exists, check if there's latest version metadata
if versionsEntry.Extended != nil {
if _, hasLatest := versionsEntry.Extended[s3_constants.ExtLatestVersionIdKey]; hasLatest {
// There is a latest version in the .versions directory, so null is not latest
isLatest = false
glog.V(2).Infof("findVersionsRecursively: null version for %s is not latest due to versioned objects", objectKey)
}
}
}
etag := s3a.calculateETagFromChunks(entry.Chunks)
versionEntry := &VersionEntry{
Key: objectKey,
VersionId: "null",
IsLatest: isLatest,
LastModified: time.Unix(entry.Attributes.Mtime, 0),
ETag: etag,
Size: int64(entry.Attributes.FileSize),
Owner: CanonicalUser{ID: "unknown", DisplayName: "unknown"},
StorageClass: "STANDARD",
}
*allVersions = append(*allVersions, versionEntry)
}
}
return nil
}
// getObjectVersionList returns all versions of a specific object
func (s3a *S3ApiServer) getObjectVersionList(bucket, object string) ([]*ObjectVersion, error) {
var versions []*ObjectVersion
@@ -386,9 +287,6 @@ func (s3a *S3ApiServer) getObjectVersionList(bucket, object string) ([]*ObjectVe
glog.V(2).Infof("getObjectVersionList: found %d entries in versions directory", len(entries))
// Use a map to detect and prevent duplicate version IDs
seenVersionIds := make(map[string]bool)
for i, entry := range entries {
if entry.Extended == nil {
glog.V(2).Infof("getObjectVersionList: entry %d has no Extended metadata, skipping", i)
@@ -403,13 +301,6 @@ func (s3a *S3ApiServer) getObjectVersionList(bucket, object string) ([]*ObjectVe
versionId := string(versionIdBytes)
// Check for duplicate version IDs and skip if already seen
if seenVersionIds[versionId] {
glog.Warningf("getObjectVersionList: duplicate version ID %s detected for object %s/%s, skipping", versionId, bucket, object)
continue
}
seenVersionIds[versionId] = true
// Check if this version is the latest by comparing with directory metadata
isLatest := (versionId == latestVersionId)
@@ -440,9 +331,12 @@ func (s3a *S3ApiServer) getObjectVersionList(bucket, object string) ([]*ObjectVe
versions = append(versions, version)
}
// Don't sort here - let the main listObjectVersions function handle sorting consistently
// Sort by modification time (newest first)
sort.Slice(versions, func(i, j int) bool {
return versions[i].LastModified.After(versions[j].LastModified)
})
glog.V(2).Infof("getObjectVersionList: returning %d total versions for %s/%s (after deduplication from %d entries)", len(versions), bucket, object, len(entries))
glog.V(2).Infof("getObjectVersionList: returning %d total versions for %s/%s", len(versions), bucket, object)
for i, version := range versions {
glog.V(2).Infof("getObjectVersionList: version %d: %s (isLatest=%v, isDeleteMarker=%v)", i, version.VersionId, version.IsLatest, version.IsDeleteMarker)
}
@@ -472,16 +366,6 @@ func (s3a *S3ApiServer) getSpecificObjectVersion(bucket, object, versionId strin
return s3a.getEntry(path.Join(s3a.option.BucketsPath, bucket), strings.TrimPrefix(object, "/"))
}
if versionId == "null" {
// "null" version ID refers to pre-versioning objects stored as regular files
bucketDir := s3a.option.BucketsPath + "/" + bucket
entry, err := s3a.getEntry(bucketDir, object)
if err != nil {
return nil, fmt.Errorf("null version object %s not found: %v", object, err)
}
return entry, nil
}
// Get specific version from .versions directory
versionsDir := s3a.getVersionedObjectDir(bucket, object)
versionFile := s3a.getVersionFileName(versionId)
@@ -500,32 +384,6 @@ func (s3a *S3ApiServer) deleteSpecificObjectVersion(bucket, object, versionId st
return fmt.Errorf("version ID is required for version-specific deletion")
}
if versionId == "null" {
// Delete "null" version (pre-versioning object stored as regular file)
bucketDir := s3a.option.BucketsPath + "/" + bucket
cleanObject := strings.TrimPrefix(object, "/")
// Check if the object exists
_, err := s3a.getEntry(bucketDir, cleanObject)
if err != nil {
// Object doesn't exist - this is OK for delete operations (idempotent)
glog.V(2).Infof("deleteSpecificObjectVersion: null version object %s already deleted or doesn't exist", cleanObject)
return nil
}
// Delete the regular file
deleteErr := s3a.rm(bucketDir, cleanObject, true, false)
if deleteErr != nil {
// Check if file was already deleted by another process
if _, checkErr := s3a.getEntry(bucketDir, cleanObject); checkErr != nil {
// File doesn't exist anymore, deletion was successful
return nil
}
return fmt.Errorf("failed to delete null version %s: %v", cleanObject, deleteErr)
}
return nil
}
versionsDir := s3a.getVersionedObjectDir(bucket, object)
versionFile := s3a.getVersionFileName(versionId)
@@ -535,120 +393,16 @@ func (s3a *S3ApiServer) deleteSpecificObjectVersion(bucket, object, versionId st
return fmt.Errorf("version %s not found: %v", versionId, err)
}
// Check if this is the latest version before deleting
versionsEntry, dirErr := s3a.getEntry(path.Join(s3a.option.BucketsPath, bucket), object+".versions")
isLatestVersion := false
if dirErr == nil && versionsEntry.Extended != nil {
if latestVersionIdBytes, hasLatest := versionsEntry.Extended[s3_constants.ExtLatestVersionIdKey]; hasLatest {
isLatestVersion = (string(latestVersionIdBytes) == versionId)
}
}
// Delete the version file
// Version exists, delete it
deleteErr := s3a.rm(versionsDir, versionFile, true, false)
if deleteErr != nil {
// Check if file was already deleted by another process
if _, checkErr := s3a.getEntry(versionsDir, versionFile); checkErr != nil {
// File doesn't exist anymore, deletion was successful
} else {
return fmt.Errorf("failed to delete version %s: %v", versionId, deleteErr)
return nil
}
return fmt.Errorf("failed to delete version %s: %v", versionId, deleteErr)
}
// If we deleted the latest version, update the .versions directory metadata to point to the new latest
if isLatestVersion {
err := s3a.updateLatestVersionAfterDeletion(bucket, object)
if err != nil {
glog.Warningf("deleteSpecificObjectVersion: failed to update latest version after deletion: %v", err)
// Don't return error since the deletion was successful
}
}
return nil
}
// updateLatestVersionAfterDeletion finds the new latest version after deleting the current latest
func (s3a *S3ApiServer) updateLatestVersionAfterDeletion(bucket, object string) error {
bucketDir := s3a.option.BucketsPath + "/" + bucket
cleanObject := strings.TrimPrefix(object, "/")
versionsObjectPath := cleanObject + ".versions"
versionsDir := bucketDir + "/" + versionsObjectPath
glog.V(1).Infof("updateLatestVersionAfterDeletion: updating latest version for %s/%s, listing %s", bucket, object, versionsDir)
// List all remaining version files in the .versions directory
entries, _, err := s3a.list(versionsDir, "", "", false, 1000)
if err != nil {
glog.Errorf("updateLatestVersionAfterDeletion: failed to list versions in %s: %v", versionsDir, err)
return fmt.Errorf("failed to list versions: %v", err)
}
glog.V(1).Infof("updateLatestVersionAfterDeletion: found %d entries in %s", len(entries), versionsDir)
// Find the most recent remaining version (latest timestamp in version ID)
var latestVersionId string
var latestVersionFileName string
for _, entry := range entries {
if entry.Extended == nil {
continue
}
versionIdBytes, hasVersionId := entry.Extended[s3_constants.ExtVersionIdKey]
if !hasVersionId {
continue
}
versionId := string(versionIdBytes)
// Skip delete markers when finding latest content version
isDeleteMarkerBytes, _ := entry.Extended[s3_constants.ExtDeleteMarkerKey]
if string(isDeleteMarkerBytes) == "true" {
continue
}
// Compare version IDs chronologically (our version IDs start with timestamp)
if latestVersionId == "" || versionId > latestVersionId {
glog.V(1).Infof("updateLatestVersionAfterDeletion: found newer version %s (file: %s)", versionId, entry.Name)
latestVersionId = versionId
latestVersionFileName = entry.Name
} else {
glog.V(1).Infof("updateLatestVersionAfterDeletion: skipping older version %s", versionId)
}
}
// Update the .versions directory metadata
versionsEntry, err := s3a.getEntry(bucketDir, versionsObjectPath)
if err != nil {
return fmt.Errorf("failed to get .versions directory: %v", err)
}
if versionsEntry.Extended == nil {
versionsEntry.Extended = make(map[string][]byte)
}
if latestVersionId != "" {
// Update metadata to point to new latest version
versionsEntry.Extended[s3_constants.ExtLatestVersionIdKey] = []byte(latestVersionId)
versionsEntry.Extended[s3_constants.ExtLatestVersionFileNameKey] = []byte(latestVersionFileName)
glog.V(2).Infof("updateLatestVersionAfterDeletion: new latest version for %s/%s is %s", bucket, object, latestVersionId)
} else {
// No versions left, remove latest version metadata
delete(versionsEntry.Extended, s3_constants.ExtLatestVersionIdKey)
delete(versionsEntry.Extended, s3_constants.ExtLatestVersionFileNameKey)
glog.V(2).Infof("updateLatestVersionAfterDeletion: no versions left for %s/%s", bucket, object)
}
// Update the .versions directory entry
err = s3a.mkFile(bucketDir, versionsObjectPath, versionsEntry.Chunks, func(updatedEntry *filer_pb.Entry) {
updatedEntry.Extended = versionsEntry.Extended
updatedEntry.Attributes = versionsEntry.Attributes
updatedEntry.Chunks = versionsEntry.Chunks
})
if err != nil {
return fmt.Errorf("failed to update .versions directory metadata: %v", err)
}
return nil
}
@@ -696,56 +450,24 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http
// getLatestObjectVersion finds the latest version of an object by reading .versions directory metadata
func (s3a *S3ApiServer) getLatestObjectVersion(bucket, object string) (*filer_pb.Entry, error) {
bucketDir := s3a.option.BucketsPath + "/" + bucket
cleanObject := strings.TrimPrefix(object, "/")
versionsObjectPath := cleanObject + ".versions"
versionsObjectPath := object + ".versions"
// Get the .versions directory entry to read latest version metadata
versionsEntry, err := s3a.getEntry(bucketDir, versionsObjectPath)
if err != nil {
// .versions directory doesn't exist - this can happen for objects that existed
// before versioning was enabled on the bucket. Fall back to checking for a
// regular (non-versioned) object file.
glog.V(2).Infof("getLatestObjectVersion: no .versions directory for %s/%s, checking for pre-versioning object", bucket, object)
regularEntry, regularErr := s3a.getEntry(bucketDir, cleanObject)
if regularErr != nil {
return nil, fmt.Errorf("failed to get %s/%s .versions directory and no regular object found: %w", bucket, cleanObject, err)
}
glog.V(2).Infof("getLatestObjectVersion: found pre-versioning object for %s/%s", bucket, cleanObject)
return regularEntry, nil
return nil, fmt.Errorf("failed to get .versions directory: %w", err)
}
// Check if directory has latest version metadata
if versionsEntry.Extended == nil {
// No metadata means all versioned objects have been deleted.
// Fall back to checking for a pre-versioning object.
glog.V(2).Infof("getLatestObjectVersion: no Extended metadata in .versions directory for %s/%s, checking for pre-versioning object", bucket, cleanObject)
regularEntry, regularErr := s3a.getEntry(bucketDir, cleanObject)
if regularErr != nil {
return nil, fmt.Errorf("no version metadata in .versions directory and no regular object found for %s/%s", bucket, cleanObject)
}
glog.V(2).Infof("getLatestObjectVersion: found pre-versioning object for %s/%s (no Extended metadata case)", bucket, cleanObject)
return regularEntry, nil
return nil, fmt.Errorf("no version metadata found in .versions directory for %s/%s", bucket, object)
}
latestVersionIdBytes, hasLatestVersionId := versionsEntry.Extended[s3_constants.ExtLatestVersionIdKey]
latestVersionFileBytes, hasLatestVersionFile := versionsEntry.Extended[s3_constants.ExtLatestVersionFileNameKey]
if !hasLatestVersionId || !hasLatestVersionFile {
// No version metadata means all versioned objects have been deleted.
// Fall back to checking for a pre-versioning object.
glog.V(2).Infof("getLatestObjectVersion: no version metadata in .versions directory for %s/%s, checking for pre-versioning object", bucket, object)
regularEntry, regularErr := s3a.getEntry(bucketDir, cleanObject)
if regularErr != nil {
return nil, fmt.Errorf("no version metadata in .versions directory and no regular object found for %s/%s", bucket, cleanObject)
}
glog.V(2).Infof("getLatestObjectVersion: found pre-versioning object for %s/%s after version deletion", bucket, cleanObject)
return regularEntry, nil
return nil, fmt.Errorf("incomplete latest version metadata in .versions directory for %s/%s", bucket, object)
}
latestVersionId := string(latestVersionIdBytes)
+1 -1
View File
@@ -8,7 +8,7 @@ import (
var (
MAJOR_VERSION = int32(3)
MINOR_VERSION = int32(95)
MINOR_VERSION = int32(94)
VERSION_NUMBER = fmt.Sprintf("%d.%02d", MAJOR_VERSION, MINOR_VERSION)
VERSION = util.SizeLimit + " " + VERSION_NUMBER
COMMIT = ""