diff --git a/test/s3/tigris_compat/.gitignore b/test/s3/tigris_compat/.gitignore index 8ff527ae7c..7a7472f9b0 100644 --- a/test/s3/tigris_compat/.gitignore +++ b/test/s3/tigris_compat/.gitignore @@ -4,3 +4,4 @@ compat-runner results.json weed-test.log weed-server.pid +weed-server.started diff --git a/test/s3/tigris_compat/Makefile b/test/s3/tigris_compat/Makefile index a31f818ca3..3254e9d591 100644 --- a/test/s3/tigris_compat/Makefile +++ b/test/s3/tigris_compat/Makefile @@ -6,6 +6,7 @@ SUITE_REPO := https://github.com/tigrisdata-community/s3-api-compat-tests.git SUITE_COMMIT := e846275e489ec30b324344d462e698252c3a1f67 SUITE_DIR := s3-api-compat-tests +SUITE_STAMP := $(SUITE_DIR)/.seaweedfs-prepared WEED_BINARY := ../../../weed/weed_binary S3_PORT ?= 8335 @@ -16,6 +17,11 @@ WEBDAV_PORT ?= 7335 ADMIN_PORT ?= 23647 SERVER_DIR := ./test-volume-data S3_CONFIG := ./test_s3.json +# Succeeds while weed-server.pid names the server start-server launched: the +# process at that pid must have the start time recorded with it, so a process +# that reused the pid is left alone. +OWN_SERVER_RUNNING = [ -f weed-server.pid ] && [ -s weed-server.started ] && \ + [ "$$(ps -p "$$(cat weed-server.pid)" -o lstart= 2>/dev/null)" = "$$(cat weed-server.started)" ] .DEFAULT_GOAL := help @@ -39,22 +45,28 @@ build-weed: @cd ../../../weed && go build -o weed_binary . build-suite: - @if [ "$$(git -C $(SUITE_DIR) rev-parse HEAD 2>/dev/null)" != "$(SUITE_COMMIT)" ]; then \ + @want="$(SUITE_COMMIT) $$(git hash-object seaweedfs-provider.patch)"; \ + if [ "$$(cat $(SUITE_STAMP) 2>/dev/null)" != "$$want" ]; then \ rm -rf $(SUITE_DIR) && mkdir -p $(SUITE_DIR) && \ git -C $(SUITE_DIR) init -q && \ git -C $(SUITE_DIR) fetch -q --depth 1 $(SUITE_REPO) $(SUITE_COMMIT) && \ git -C $(SUITE_DIR) checkout -q FETCH_HEAD && \ - git -C $(SUITE_DIR) apply ../seaweedfs-provider.patch; \ + git -C $(SUITE_DIR) apply ../seaweedfs-provider.patch && \ + echo "$$want" > $(SUITE_STAMP); \ fi @cd $(SUITE_DIR) && go build -o ../compat-runner . start-server: build-weed - @rm -f weed-server.pid - @mkdir -p $(SERVER_DIR) - @if curl -sS -o /dev/null http://127.0.0.1:$(S3_PORT)/ >/dev/null 2>&1; then \ + @if curl -sS --max-time 2 -o /dev/null http://127.0.0.1:$(S3_PORT)/ >/dev/null 2>&1; then \ echo "port $(S3_PORT) is already served by another process; refusing to test against it"; \ exit 1; \ fi + @if $(OWN_SERVER_RUNNING); then \ + echo "the server this Makefile started (pid $$(cat weed-server.pid)) is still running; run make stop-server first"; \ + exit 1; \ + fi + @rm -rf weed-server.pid weed-server.started $(SERVER_DIR) + @mkdir -p $(SERVER_DIR) @$(WEED_BINARY) mini \ -dir=$(SERVER_DIR) \ -ip=127.0.0.1 \ @@ -68,14 +80,15 @@ start-server: build-weed -s3.port=$(S3_PORT) \ -s3.config=$(S3_CONFIG) \ -master.peers=none \ - > weed-test.log 2>&1 & echo $$! > weed-server.pid + > weed-test.log 2>&1 & echo $$! > weed-server.pid; \ + ps -p $$! -o lstart= > weed-server.started @for i in $$(seq 1 60); do \ kill -0 $$(cat weed-server.pid) 2>/dev/null || { \ echo "SeaweedFS server exited during startup"; \ cat weed-test.log; \ exit 1; \ }; \ - if curl -sS -o /dev/null http://127.0.0.1:$(S3_PORT)/ >/dev/null 2>&1; then \ + if curl -sS --max-time 2 -o /dev/null http://127.0.0.1:$(S3_PORT)/ >/dev/null 2>&1; then \ echo "SeaweedFS S3 server is ready on port $(S3_PORT)"; \ exit 0; \ fi; \ @@ -88,12 +101,15 @@ start-server: build-weed stop-server: @if [ -f weed-server.pid ]; then \ - pid=$$(cat weed-server.pid); \ - if ps -p $$pid -o comm= 2>/dev/null | grep -q weed; then \ - kill $$pid 2>/dev/null || true; \ - sleep 2; \ + if $(OWN_SERVER_RUNNING); then \ + kill $$(cat weed-server.pid) 2>/dev/null || true; \ + for i in $$(seq 1 30); do $(OWN_SERVER_RUNNING) || break; sleep 1; done; \ + if $(OWN_SERVER_RUNNING); then \ + echo "the server this Makefile started (pid $$(cat weed-server.pid)) is still running after 30s; keeping weed-server.pid"; \ + exit 1; \ + fi; \ fi; \ - rm -f weed-server.pid; \ + rm -f weed-server.pid weed-server.started; \ fi run-suite: build-suite @@ -104,12 +120,12 @@ run-suite: build-suite python3 check_results.py results.json expected_failures.txt test-with-server: - @$(MAKE) start-server; START_RESULT=$$?; \ - if [ $$START_RESULT -eq 0 ]; then $(MAKE) run-suite; TEST_RESULT=$$?; else TEST_RESULT=$$START_RESULT; fi; \ + @$(MAKE) start-server || exit $$?; \ + $(MAKE) run-suite; TEST_RESULT=$$?; \ $(MAKE) stop-server; exit $$TEST_RESULT logs: @test -f weed-test.log && tail -100 weed-test.log || echo "No log file found" clean: stop-server - @rm -rf $(SERVER_DIR) $(SUITE_DIR) compat-runner results.json weed-test.log weed-server.pid + @rm -rf $(SERVER_DIR) $(SUITE_DIR) compat-runner results.json weed-test.log weed-server.pid weed-server.started diff --git a/test/s3/tigris_compat/check_results.py b/test/s3/tigris_compat/check_results.py index 3d81168358..55a01ccc76 100644 --- a/test/s3/tigris_compat/check_results.py +++ b/test/s3/tigris_compat/check_results.py @@ -1,20 +1,21 @@ #!/usr/bin/env python3 """Compare an s3-api-compat-tests JSON report with expected_failures.txt. -Fails on any test that did not pass and is not listed, and on any listed test -that now passes or did not run, so the list only ever shrinks. +Fails on any test that did not pass and is not listed with the status it +ended with, and on any listed test that now passes or did not run, so the list +only ever shrinks. """ import json import sys def load_expected(path): - expected = set() + expected = {} with open(path) as f: for line in f: - name = line.split("#", 1)[0].strip() - if name: - expected.add(name) + fields = line.split("#", 1)[0].split() + if fields: + expected[fields[0]] = fields[1] if len(fields) > 1 else "fail" return expected @@ -41,10 +42,12 @@ def main(): fixed.append(name) elif name not in expected: regressions.append(f"{name} [{status}]: {detail}") + elif status != expected[name]: + regressions.append(f"{name} [{status}, listed as {expected[name]}]: {detail}") else: print(f"expected failure: {name} [{status}]") - missing = sorted(expected - seen) + missing = sorted(set(expected) - seen) print(f"{len(results)} tests, {len(regressions)} unexpected failures, " f"{len(fixed)} unexpected passes, {len(missing)} listed but not run") for line in regressions: diff --git a/test/s3/tigris_compat/expected_failures.txt b/test/s3/tigris_compat/expected_failures.txt index 469e2f09b0..6ca5b205e0 100644 --- a/test/s3/tigris_compat/expected_failures.txt +++ b/test/s3/tigris_compat/expected_failures.txt @@ -1,6 +1,8 @@ # Tests from s3-api-compat-tests that SeaweedFS is expected to fail. # check_results.py fails CI on any other failure, and on any test listed here # that passes, so remove an entry together with the fix that makes it pass. +# An entry expects the test to fail; " error" expects its setup to fail +# before the call it checks, and any other status is a regression. # AWS answers GetBucketPolicyStatus on a bucket without a policy with # NoSuchBucketPolicy, and so does SeaweedFS; the suite expects a status. @@ -14,9 +16,3 @@ RestoreObject # starts with the request's bucket, as bucket/key, because AWS documents both # spellings. A leading slash is dropped since filer paths cannot hold it. RenameObject_SourceIsLiteralKey - -# Not yet fixed in SeaweedFS. -PutBucketAccelerateConfiguration -GetBucketAccelerateConfiguration -PutBucketNotificationConfiguration -PutObjectTagging_VersionId diff --git a/weed/s3api/s3_action_resolver.go b/weed/s3api/s3_action_resolver.go index 7559b55f15..52d6a282a3 100644 --- a/weed/s3api/s3_action_resolver.go +++ b/weed/s3api/s3_action_resolver.go @@ -109,6 +109,10 @@ var bucketQueryActions = map[string]map[string]string{ http.MethodGet: s3_constants.S3_ACTION_GET_BUCKET_NOTIFICATION, http.MethodPut: s3_constants.S3_ACTION_PUT_BUCKET_NOTIFICATION, }, + "accelerate": { + http.MethodGet: s3_constants.S3_ACTION_GET_BUCKET_ACCELERATE, + http.MethodPut: s3_constants.S3_ACTION_PUT_BUCKET_ACCELERATE, + }, "object-lock": { http.MethodGet: s3_constants.S3_ACTION_GET_BUCKET_OBJECT_LOCK, http.MethodPut: s3_constants.S3_ACTION_PUT_BUCKET_OBJECT_LOCK, diff --git a/weed/s3api/s3_action_resolver_test.go b/weed/s3api/s3_action_resolver_test.go index 8d5f9f277b..8791173a60 100644 --- a/weed/s3api/s3_action_resolver_test.go +++ b/weed/s3api/s3_action_resolver_test.go @@ -280,6 +280,9 @@ func TestResolveS3Action_AdminBucketSubresources(t *testing.T) { {"delete encryption", http.MethodDelete, "encryption", s3_constants.S3_ACTION_PUT_BUCKET_ENCRYPTION}, {"get requestPayment", http.MethodGet, "requestPayment", s3_constants.S3_ACTION_GET_BUCKET_REQUEST_PAYMENT}, {"put requestPayment", http.MethodPut, "requestPayment", s3_constants.S3_ACTION_PUT_BUCKET_REQUEST_PAYMENT}, + {"get accelerate", http.MethodGet, "accelerate", s3_constants.S3_ACTION_GET_BUCKET_ACCELERATE}, + {"put accelerate", http.MethodPut, "accelerate", s3_constants.S3_ACTION_PUT_BUCKET_ACCELERATE}, + {"put notification", http.MethodPut, "notification", s3_constants.S3_ACTION_PUT_BUCKET_NOTIFICATION}, {"get publicAccessBlock", http.MethodGet, "publicAccessBlock", s3_constants.S3_ACTION_GET_BUCKET_PUBLIC_ACCESS_BLOCK}, {"put publicAccessBlock", http.MethodPut, "publicAccessBlock", s3_constants.S3_ACTION_PUT_BUCKET_PUBLIC_ACCESS_BLOCK}, {"delete publicAccessBlock", http.MethodDelete, "publicAccessBlock", s3_constants.S3_ACTION_PUT_BUCKET_PUBLIC_ACCESS_BLOCK}, diff --git a/weed/s3api/s3_constants/s3_action_strings.go b/weed/s3api/s3_constants/s3_action_strings.go index 124a29e5d2..d71ccfc46c 100644 --- a/weed/s3api/s3_constants/s3_action_strings.go +++ b/weed/s3api/s3_constants/s3_action_strings.go @@ -87,6 +87,10 @@ const ( S3_ACTION_GET_BUCKET_NOTIFICATION = "s3:GetBucketNotification" S3_ACTION_PUT_BUCKET_NOTIFICATION = "s3:PutBucketNotification" + // Bucket transfer acceleration + S3_ACTION_GET_BUCKET_ACCELERATE = "s3:GetAccelerateConfiguration" + S3_ACTION_PUT_BUCKET_ACCELERATE = "s3:PutAccelerateConfiguration" + // Bucket object lock operations S3_ACTION_GET_BUCKET_OBJECT_LOCK = "s3:GetBucketObjectLockConfiguration" S3_ACTION_PUT_BUCKET_OBJECT_LOCK = "s3:PutBucketObjectLockConfiguration" diff --git a/weed/s3api/s3api_bucket_handlers_misc.go b/weed/s3api/s3api_bucket_handlers_misc.go index 798257a8e1..d51bb6792b 100644 --- a/weed/s3api/s3api_bucket_handlers_misc.go +++ b/weed/s3api/s3api_bucket_handlers_misc.go @@ -12,9 +12,9 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" ) -// putBucketRequestPaymentMaxBodyBytes caps the request body for PutBucketRequestPayment -// to prevent DoS via large payloads. The valid payload is a few hundred bytes. -const putBucketRequestPaymentMaxBodyBytes = 64 * 1024 +// putBucketMiscConfigMaxBodyBytes caps the request body of the configuration +// PUTs in this file; a valid payload is a few hundred bytes. +const putBucketMiscConfigMaxBodyBytes = 64 * 1024 type policyStatusResponse struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ PolicyStatus"` @@ -34,6 +34,19 @@ type notificationConfigurationResponse struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NotificationConfiguration"` } +type accelerateConfigurationRequest struct { + XMLName xml.Name `xml:"AccelerateConfiguration"` + Status string `xml:"Status"` +} + +type notificationConfigurationRequest struct { + XMLName xml.Name `xml:"NotificationConfiguration"` + Topics []struct{} `xml:"TopicConfiguration"` + Queues []struct{} `xml:"QueueConfiguration"` + Functions []struct{} `xml:"CloudFunctionConfiguration"` + EventBridge []struct{} `xml:"EventBridgeConfiguration"` +} + // GetBucketPolicyStatusHandler reports whether the bucket policy grants public access. // https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketPolicyStatus.html func (s3a *S3ApiServer) GetBucketPolicyStatusHandler(w http.ResponseWriter, r *http.Request) { @@ -92,7 +105,7 @@ func (s3a *S3ApiServer) PutBucketRequestPaymentHandler(w http.ResponseWriter, r } defer r.Body.Close() - body, err := readRequestBody(r, putBucketRequestPaymentMaxBodyBytes) + body, err := readRequestBody(r, putBucketMiscConfigMaxBodyBytes) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML) return @@ -123,6 +136,67 @@ func (s3a *S3ApiServer) GetBucketAccelerateConfigurationHandler(w http.ResponseW writeSuccessResponseXML(w, r, accelerateConfigurationResponse{Status: "Suspended"}) } +// PutBucketAccelerateConfigurationHandler accepts Status=Suspended, the state +// GetBucketAccelerateConfigurationHandler always reports; Enabled is not implemented. +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAccelerateConfiguration.html +func (s3a *S3ApiServer) PutBucketAccelerateConfigurationHandler(w http.ResponseWriter, r *http.Request) { + bucket, _ := s3_constants.GetBucketAndObject(r) + if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, err) + return + } + + defer r.Body.Close() + body, err := readRequestBody(r, putBucketMiscConfigMaxBodyBytes) + if err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML) + return + } + var cfg accelerateConfigurationRequest + if err := xml.Unmarshal(body, &cfg); err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML) + return + } + + switch cfg.Status { + case "Suspended": + writeSuccessResponseEmpty(w, r) + case "Enabled": + s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) + default: + s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML) + } +} + +// PutBucketNotificationConfigurationHandler accepts an empty configuration, the +// one GetBucketNotificationConfigurationHandler always reports; any destination +// is not implemented. +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketNotificationConfiguration.html +func (s3a *S3ApiServer) PutBucketNotificationConfigurationHandler(w http.ResponseWriter, r *http.Request) { + bucket, _ := s3_constants.GetBucketAndObject(r) + if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, err) + return + } + + defer r.Body.Close() + body, err := readRequestBody(r, putBucketMiscConfigMaxBodyBytes) + if err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML) + return + } + var cfg notificationConfigurationRequest + if err := xml.Unmarshal(body, &cfg); err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML) + return + } + if len(cfg.Topics)+len(cfg.Queues)+len(cfg.Functions)+len(cfg.EventBridge) > 0 { + s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) + return + } + writeSuccessResponseEmpty(w, r) +} + // GetBucketLoggingHandler returns an empty BucketLoggingStatus (logging disabled). // https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLogging.html func (s3a *S3ApiServer) GetBucketLoggingHandler(w http.ResponseWriter, r *http.Request) { diff --git a/weed/s3api/s3api_bucket_handlers_misc_test.go b/weed/s3api/s3api_bucket_handlers_misc_test.go index 97479748f3..1a0fdb47b6 100644 --- a/weed/s3api/s3api_bucket_handlers_misc_test.go +++ b/weed/s3api/s3api_bucket_handlers_misc_test.go @@ -710,3 +710,52 @@ func TestPutBucketAclKeepsOwner(t *testing.T) { require.Equal(t, "team-a", aws.StringValue(grants[0].Grantee.ID)) } } + +func TestPutBucketAccelerateConfiguration(t *testing.T) { + cases := []struct { + name string + body string + want int + }{ + {"suspended", `Suspended`, http.StatusOK}, + {"enabled", `Enabled`, http.StatusNotImplemented}, + {"invalid status", `On`, http.StatusBadRequest}, + {"malformed", ``, http.StatusBadRequest}, + {"wrong root", `Suspended`, http.StatusBadRequest}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s3a := newMiscTestServer(t, "b") + rec := httptest.NewRecorder() + s3a.PutBucketAccelerateConfigurationHandler(rec, newBucketRequest(http.MethodPut, "b", "accelerate=", tc.body)) + if rec.Code != tc.want { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, tc.want, rec.Body.String()) + } + }) + } +} + +func TestPutBucketNotificationConfiguration(t *testing.T) { + cases := []struct { + name string + body string + want int + }{ + {"empty", ``, http.StatusOK}, + {"topic", `arn:aws:sns:us-east-1:1:ts3:ObjectCreated:*`, http.StatusNotImplemented}, + {"queue", `arn:aws:sqs:us-east-1:1:qs3:ObjectCreated:*`, http.StatusNotImplemented}, + {"eventbridge", ``, http.StatusNotImplemented}, + {"malformed", ``, http.StatusBadRequest}, + {"wrong root", ``, http.StatusBadRequest}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s3a := newMiscTestServer(t, "b") + rec := httptest.NewRecorder() + s3a.PutBucketNotificationConfigurationHandler(rec, newBucketRequest(http.MethodPut, "b", "notification=", tc.body)) + if rec.Code != tc.want { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, tc.want, rec.Body.String()) + } + }) + } +} diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index e53ea0b6d1..12f370f62c 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -1158,10 +1158,14 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketPolicyStatusHandler, ACTION_READ)), "GET")).Queries("policyStatus", "") // GetBucketAccelerateConfiguration bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketAccelerateConfigurationHandler, ACTION_READ)), "GET")).Queries("accelerate", "") + // PutBucketAccelerateConfiguration + bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketAccelerateConfigurationHandler, ACTION_ADMIN)), "PUT")).Queries("accelerate", "") // GetBucketLogging bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLoggingHandler, ACTION_READ)), "GET")).Queries("logging", "") // GetBucketNotificationConfiguration bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketNotificationConfigurationHandler, ACTION_READ)), "GET")).Queries("notification", "") + // PutBucketNotificationConfiguration + bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketNotificationConfigurationHandler, ACTION_ADMIN)), "PUT")).Queries("notification", "") // GetBucketReplication bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketReplicationHandler, ACTION_READ)), "GET")).Queries("replication", "") // GetBucketWebsite