mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
s3api: report no ownership controls after DeleteBucketOwnershipControls (#12748)
* test/s3: run tigrisdata-community s3-api-compat-tests in CI Runs the suite, pinned to a commit, against weed mini through a small provider patch, and fails on any test that is not in expected_failures.txt or on a listed test that starts passing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3api: report no ownership controls after DeleteBucketOwnershipControls GetBucketOwnershipControls reports BucketOwnerEnforced for a bucket that never stored a setting, so it kept reporting it after the setting was deleted. The delete now records that it happened, and GET answers OwnershipControlsNotFoundError until ownership is put again. How objects are written after the delete is unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011UpvafvK8JpagF94MHWqwK * s3api: answer DeleteBucketOwnershipControls with 204 No Content AWS answers a successful DeleteBucketOwnershipControls with 204 and no body; this answered 200 with an empty OwnershipControls document. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aceb089bc8
commit
b1c315b837
12 files changed
+383
-11
No files matched your search
@@ -2,13 +2,16 @@ package acl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
smithyhttp "github.com/aws/smithy-go/transport/http"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -63,12 +66,22 @@ func TestBucketOwnershipControlsLifecycle(t *testing.T) {
|
||||
putObjectOwnership(t, client, bucketName, types.ObjectOwnershipBucketOwnerEnforced)
|
||||
assert.Equal(t, types.ObjectOwnershipBucketOwnerEnforced, getObjectOwnership(t, client, bucketName))
|
||||
|
||||
_, err := client.DeleteBucketOwnershipControls(context.TODO(), &s3.DeleteBucketOwnershipControlsInput{
|
||||
deleted, err := client.DeleteBucketOwnershipControls(context.TODO(), &s3.DeleteBucketOwnershipControlsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
raw, ok := awsmiddleware.GetRawResponse(deleted.ResultMetadata).(*smithyhttp.Response)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, http.StatusNoContent, raw.StatusCode)
|
||||
|
||||
assert.Equal(t, types.ObjectOwnershipBucketOwnerEnforced, getObjectOwnership(t, client, bucketName))
|
||||
_, err = client.GetBucketOwnershipControls(context.TODO(), &s3.GetBucketOwnershipControlsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "OwnershipControlsNotFoundError")
|
||||
|
||||
putObjectOwnership(t, client, bucketName, types.ObjectOwnershipBucketOwnerPreferred)
|
||||
assert.Equal(t, types.ObjectOwnershipBucketOwnerPreferred, getObjectOwnership(t, client, bucketName))
|
||||
}
|
||||
|
||||
// A bucket with nothing stored already reports BucketOwnerEnforced, so this put has
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
s3-api-compat-tests/
|
||||
test-volume-data/
|
||||
compat-runner
|
||||
results.json
|
||||
weed-test.log
|
||||
weed-server.pid
|
||||
@@ -0,0 +1,115 @@
|
||||
# Runs https://github.com/tigrisdata-community/s3-api-compat-tests against a
|
||||
# local SeaweedFS S3 gateway and compares the report with expected_failures.txt.
|
||||
|
||||
.PHONY: all test help build-weed build-suite start-server stop-server run-suite test-with-server logs clean
|
||||
|
||||
SUITE_REPO := https://github.com/tigrisdata-community/s3-api-compat-tests.git
|
||||
SUITE_COMMIT := e846275e489ec30b324344d462e698252c3a1f67
|
||||
SUITE_DIR := s3-api-compat-tests
|
||||
|
||||
WEED_BINARY := ../../../weed/weed_binary
|
||||
S3_PORT ?= 8335
|
||||
MASTER_PORT ?= 9335
|
||||
VOLUME_PORT ?= 8085
|
||||
FILER_PORT ?= 8889
|
||||
WEBDAV_PORT ?= 7335
|
||||
ADMIN_PORT ?= 23647
|
||||
SERVER_DIR := ./test-volume-data
|
||||
S3_CONFIG := ./test_s3.json
|
||||
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
all: test
|
||||
|
||||
test: test-with-server
|
||||
|
||||
help:
|
||||
@echo "s3-api-compat-tests (Tigris) against SeaweedFS"
|
||||
@echo ""
|
||||
@echo " build-weed - Build the SeaweedFS binary"
|
||||
@echo " build-suite - Fetch the suite at $(SUITE_COMMIT) and build it"
|
||||
@echo " start-server - Start weed mini with S3 on port $(S3_PORT)"
|
||||
@echo " run-suite - Run the suite and check expected_failures.txt"
|
||||
@echo " stop-server - Stop the test server"
|
||||
@echo " test-with-server - Start server, run suite, stop server"
|
||||
@echo " logs - Show server logs"
|
||||
@echo " clean - Remove test artifacts"
|
||||
|
||||
build-weed:
|
||||
@cd ../../../weed && go build -o weed_binary .
|
||||
|
||||
build-suite:
|
||||
@if [ "$$(git -C $(SUITE_DIR) rev-parse HEAD 2>/dev/null)" != "$(SUITE_COMMIT)" ]; then \
|
||||
rm -rf $(SUITE_DIR) && mkdir -p $(SUITE_DIR) && \
|
||||
git -C $(SUITE_DIR) init -q && \
|
||||
git -C $(SUITE_DIR) fetch -q --depth 1 $(SUITE_REPO) $(SUITE_COMMIT) && \
|
||||
git -C $(SUITE_DIR) checkout -q FETCH_HEAD && \
|
||||
git -C $(SUITE_DIR) apply ../seaweedfs-provider.patch; \
|
||||
fi
|
||||
@cd $(SUITE_DIR) && go build -o ../compat-runner .
|
||||
|
||||
start-server: build-weed
|
||||
@rm -f weed-server.pid
|
||||
@mkdir -p $(SERVER_DIR)
|
||||
@if curl -sS -o /dev/null http://127.0.0.1:$(S3_PORT)/ >/dev/null 2>&1; then \
|
||||
echo "port $(S3_PORT) is already served by another process; refusing to test against it"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@$(WEED_BINARY) mini \
|
||||
-dir=$(SERVER_DIR) \
|
||||
-ip=127.0.0.1 \
|
||||
-ip.bind=127.0.0.1 \
|
||||
-master.port=$(MASTER_PORT) \
|
||||
-volume.port=$(VOLUME_PORT) \
|
||||
-filer.port=$(FILER_PORT) \
|
||||
-webdav.port=$(WEBDAV_PORT) \
|
||||
-admin.port=$(ADMIN_PORT) \
|
||||
-admin.port.legacy=0 \
|
||||
-s3.port=$(S3_PORT) \
|
||||
-s3.config=$(S3_CONFIG) \
|
||||
-master.peers=none \
|
||||
> weed-test.log 2>&1 & echo $$! > weed-server.pid
|
||||
@for i in $$(seq 1 60); do \
|
||||
kill -0 $$(cat weed-server.pid) 2>/dev/null || { \
|
||||
echo "SeaweedFS server exited during startup"; \
|
||||
cat weed-test.log; \
|
||||
exit 1; \
|
||||
}; \
|
||||
if curl -sS -o /dev/null http://127.0.0.1:$(S3_PORT)/ >/dev/null 2>&1; then \
|
||||
echo "SeaweedFS S3 server is ready on port $(S3_PORT)"; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
sleep 1; \
|
||||
done; \
|
||||
echo "SeaweedFS S3 server failed to start"; \
|
||||
cat weed-test.log; \
|
||||
$(MAKE) --no-print-directory stop-server; \
|
||||
exit 1
|
||||
|
||||
stop-server:
|
||||
@if [ -f weed-server.pid ]; then \
|
||||
pid=$$(cat weed-server.pid); \
|
||||
if ps -p $$pid -o comm= 2>/dev/null | grep -q weed; then \
|
||||
kill $$pid 2>/dev/null || true; \
|
||||
sleep 2; \
|
||||
fi; \
|
||||
rm -f weed-server.pid; \
|
||||
fi
|
||||
|
||||
run-suite: build-suite
|
||||
SEAWEEDFS_S3_ENDPOINT=http://127.0.0.1:$(S3_PORT) \
|
||||
SEAWEEDFS_S3_ACCESS_KEY_ID=admin \
|
||||
SEAWEEDFS_S3_SECRET_ACCESS_KEY=admin \
|
||||
./compat-runner --providers=seaweedfs --output=results.json
|
||||
python3 check_results.py results.json expected_failures.txt
|
||||
|
||||
test-with-server:
|
||||
@$(MAKE) start-server; START_RESULT=$$?; \
|
||||
if [ $$START_RESULT -eq 0 ]; then $(MAKE) run-suite; TEST_RESULT=$$?; else TEST_RESULT=$$START_RESULT; fi; \
|
||||
$(MAKE) stop-server; exit $$TEST_RESULT
|
||||
|
||||
logs:
|
||||
@test -f weed-test.log && tail -100 weed-test.log || echo "No log file found"
|
||||
|
||||
clean: stop-server
|
||||
@rm -rf $(SERVER_DIR) $(SUITE_DIR) compat-runner results.json weed-test.log weed-server.pid
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare an s3-api-compat-tests JSON report with expected_failures.txt.
|
||||
|
||||
Fails on any test that did not pass and is not listed, and on any listed test
|
||||
that now passes or did not run, so the list only ever shrinks.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
def load_expected(path):
|
||||
expected = set()
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
name = line.split("#", 1)[0].strip()
|
||||
if name:
|
||||
expected.add(name)
|
||||
return expected
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 3:
|
||||
sys.exit("usage: check_results.py <results.json> <expected_failures.txt>")
|
||||
with open(sys.argv[1]) as f:
|
||||
report = json.load(f)
|
||||
expected = load_expected(sys.argv[2])
|
||||
|
||||
providers = report.get("providers") or []
|
||||
results = providers[0].get("results") if providers else None
|
||||
if not results:
|
||||
sys.exit("no test results in the report: the suite could not start against the server")
|
||||
|
||||
seen = set()
|
||||
regressions, fixed = [], []
|
||||
for r in results:
|
||||
name, status = r["operation"], r["status"]
|
||||
seen.add(name)
|
||||
detail = r.get("error") or r.get("notes") or ""
|
||||
if status == "pass":
|
||||
if name in expected:
|
||||
fixed.append(name)
|
||||
elif name not in expected:
|
||||
regressions.append(f"{name} [{status}]: {detail}")
|
||||
else:
|
||||
print(f"expected failure: {name} [{status}]")
|
||||
|
||||
missing = sorted(expected - seen)
|
||||
print(f"{len(results)} tests, {len(regressions)} unexpected failures, "
|
||||
f"{len(fixed)} unexpected passes, {len(missing)} listed but not run")
|
||||
for line in regressions:
|
||||
print(f"UNEXPECTED FAILURE: {line}")
|
||||
for name in fixed:
|
||||
print(f"UNEXPECTED PASS: {name} (remove it from expected_failures.txt)")
|
||||
for name in missing:
|
||||
print(f"NOT RUN: {name} (remove it from expected_failures.txt)")
|
||||
if regressions or fixed or missing:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,22 @@
|
||||
# Tests from s3-api-compat-tests that SeaweedFS is expected to fail.
|
||||
# check_results.py fails CI on any other failure, and on any test listed here
|
||||
# that passes, so remove an entry together with the fix that makes it pass.
|
||||
|
||||
# AWS answers GetBucketPolicyStatus on a bucket without a policy with
|
||||
# NoSuchBucketPolicy, and so does SeaweedFS; the suite expects a status.
|
||||
GetBucketPolicyStatus
|
||||
|
||||
# Glacier restore: SeaweedFS has no archive storage class to restore from, and
|
||||
# AWS also refuses RestoreObject on a STANDARD object (InvalidObjectState).
|
||||
RestoreObject
|
||||
|
||||
# SeaweedFS reads x-amz-rename-source as a literal key first and then, when it
|
||||
# starts with the request's bucket, as bucket/key, because AWS documents both
|
||||
# spellings. A leading slash is dropped since filer paths cannot hold it.
|
||||
RenameObject_SourceIsLiteralKey
|
||||
|
||||
# Not yet fixed in SeaweedFS.
|
||||
PutBucketAccelerateConfiguration
|
||||
GetBucketAccelerateConfiguration
|
||||
PutBucketNotificationConfiguration
|
||||
PutObjectTagging_VersionId
|
||||
@@ -0,0 +1,19 @@
|
||||
diff --git a/providers.go b/providers.go
|
||||
index 149f94c..bc7aeeb 100644
|
||||
--- a/providers.go
|
||||
+++ b/providers.go
|
||||
@@ -180,6 +180,14 @@ func (p *Provider) CreateVersionedBucket(ctx context.Context, client *s3.Client,
|
||||
// If names is non-empty, only those named providers are returned.
|
||||
func LoadProviders(names []string) []Provider {
|
||||
all := []Provider{
|
||||
+ {
|
||||
+ Name: "seaweedfs",
|
||||
+ Endpoint: os.Getenv("SEAWEEDFS_S3_ENDPOINT"),
|
||||
+ Region: envOr("SEAWEEDFS_S3_REGION", "us-east-1"),
|
||||
+ PathStyle: true,
|
||||
+ AccessKey: os.Getenv("SEAWEEDFS_S3_ACCESS_KEY_ID"),
|
||||
+ SecretKey: os.Getenv("SEAWEEDFS_S3_SECRET_ACCESS_KEY"),
|
||||
+ },
|
||||
{
|
||||
Name: "tigris",
|
||||
Endpoint: "https://t3.storage.dev",
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "admin",
|
||||
"secretKey": "admin"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Admin",
|
||||
"Read",
|
||||
"List",
|
||||
"Tagging",
|
||||
"Write"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user