From 97e1758bccfc37037309350722508a58cd9c1c14 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 31 Oct 2026 05:08:01 -0700 Subject: [PATCH] s3: auto-create bucket on POST Object uploads to a missing bucket (#12266) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * s3: auto-create bucket on POST Object uploads to a missing bucket PutObject and CreateMultipartUpload lazily create the bucket, but a signed POST Object upload answered NoSuchBucket. Mirror the PUT path: when the versioning-state lookup reports the bucket missing, run the shared handleAutoCreateBucket gate (bucket-name check, AutoCreateBucket option, admin identity) and then re-read the versioning state. The POST handler already records the verified signer identity in the request context, so the admin check and bucket-owner attribution work unchanged. * test: assert the POST auto-create upload lands, not just non-404 The missing-bucket POST test passed whenever the handler created the bucket, even if the upload then failed. Drive the full path through the existing putToFiler fakes — assigned volume accepts the chunk, object entry and payload land — and assert the 201 success response. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3: record verified post-policy signer's account id The POST policy path authenticates in the handler, after the internal account header was already scrubbed, so auto-created buckets and the uploaded objects stored no canonical owner. * test: assert POST auto-create records bucket owner, keep writes applied in denied case --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../s3api/s3api_object_handlers_postpolicy.go | 18 +++- ...i_object_handlers_postpolicy_authz_test.go | 101 ++++++++++++++++++ 2 files changed, 115 insertions(+), 4 deletions(-) diff --git a/weed/s3api/s3api_object_handlers_postpolicy.go b/weed/s3api/s3api_object_handlers_postpolicy.go index 8ebcc71097..78fdedd8d0 100644 --- a/weed/s3api/s3api_object_handlers_postpolicy.go +++ b/weed/s3api/s3api_object_handlers_postpolicy.go @@ -88,6 +88,9 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R } if identity != nil { r = r.WithContext(recordIdentityInContext(r, identity)) + if identity.Account != nil { + r.Header.Set(s3_constants.AmzAccountId, identity.Account.Id) + } } policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy")) @@ -156,12 +159,19 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R versioningState, err := s3a.getVersioningState(bucket) if err != nil { if errors.Is(err, filer_pb.ErrNotFound) { - s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket) + // Auto-create bucket if it doesn't exist (requires Admin permission) + if !s3a.handleAutoCreateBucket(w, r, bucket, "PostPolicyBucketHandler") { + return + } + // Re-fetch versioning state to handle race conditions where + // another process might have created the bucket with versioning enabled. + versioningState, err = s3a.getVersioningState(bucket) + } + if err != nil { + glog.Errorf("PostPolicyBucketHandler: versioning state for bucket %s: %v", bucket, err) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } - glog.Errorf("PostPolicyBucketHandler: versioning state for bucket %s: %v", bucket, err) - s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) - return } objectLockEnabled, err := s3a.isObjectLockEnabled(bucket) if err != nil && !errors.Is(err, filer_pb.ErrNotFound) { diff --git a/weed/s3api/s3api_object_handlers_postpolicy_authz_test.go b/weed/s3api/s3api_object_handlers_postpolicy_authz_test.go index a0e0f7e4ce..c341979181 100644 --- a/weed/s3api/s3api_object_handlers_postpolicy_authz_test.go +++ b/weed/s3api/s3api_object_handlers_postpolicy_authz_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" @@ -37,6 +38,9 @@ const ( postPolicySecretKey = "secret-key-for-policy-only" postReadAccessKey = "AKIAPOLICYREAD0000" postReadSecretKey = "secret-key-for-policy-read" + + postAdminAccessKey = "AKIAADMINTEST000" + postAdminSecretKey = "secret-key-for-admin-tests" ) // newPostAuthzServer builds an S3ApiServer whose non-admin "tester" identity @@ -394,3 +398,100 @@ func TestPostPolicyAttachedPolicyDeny(t *testing.T) { "POST from a read-only identity must stay AccessDenied, body: %s", rec.Body.String()) assert.Contains(t, rec.Body.String(), "AccessDenied") } + +// newPostAutoCreateServer builds an S3ApiServer on the fake filer with +// auto-create enabled, an "admin" identity allowed to create buckets, and the +// non-admin "tester" writer. +func newPostAutoCreateServer(t *testing.T, filer *ambiguousPutFiler) *S3ApiServer { + t.Helper() + + iam := &IdentityAccessManagement{ + hashes: make(map[string]*sync.Pool), + hashCounters: make(map[string]*int32), + } + err := iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{{ + Name: "admin", + Account: &iam_pb.Account{Id: "000000000000", DisplayName: "admin"}, + Credentials: []*iam_pb.Credential{{AccessKey: postAdminAccessKey, SecretKey: postAdminSecretKey}}, + Actions: []string{"Admin"}, + }, { + Name: "tester", + Account: &iam_pb.Account{Id: "000000000000", DisplayName: "tester"}, + Credentials: []*iam_pb.Credential{{AccessKey: postAuthzAccessKey, SecretKey: postAuthzSecretKey}}, + Actions: []string{"Read:" + postAuthzBucket, "Write:" + postAuthzBucket, "List:" + postAuthzBucket}, + }}, + }) + require.NoError(t, err) + + s3a := newFailoverTestServer(t, startFakeFiler(t, filer)) + s3a.iam = iam + s3a.option.BucketsPath = "/buckets" + s3a.option.AutoCreateBucket = true + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + return s3a +} + +// A signed POST Object upload to a bucket that does not exist must lazy-create +// it the way PutObject does: an admin signer is allowed to create the bucket +// and the upload proceeds against it, landing the object entry and its chunk +// instead of answering NoSuchBucket. +func TestPostPolicyBucketHandlerAutoCreatesMissingBucket(t *testing.T) { + volume := startFakeVolumeServer(t) + filer := &ambiguousPutFiler{ + volume: volume, + entries: map[string]*filer_pb.Entry{}, + apply: true, + } + s3a := newPostAutoCreateServer(t, filer) + + rec := httptest.NewRecorder() + s3a.PostPolicyBucketHandler(rec, newSignedPostRequestFor(t, "uploads/pic.png", + postAdminAccessKey, postAdminSecretKey, map[string]string{"success_action_status": "201"})) + + assert.Equal(t, http.StatusCreated, rec.Code, + "POST to a missing bucket must complete the upload after creating it, body: %s", rec.Body.String()) + assert.Contains(t, rec.Body.String(), "uploads/pic.png") + + filer.mu.Lock() + defer filer.mu.Unlock() + bucketEntry := filer.entries["/buckets/"+postAuthzBucket] + require.NotNil(t, bucketEntry, "POST to a missing bucket must create it like PUT") + assert.True(t, bucketEntry.IsDirectory) + assert.Equal(t, []byte("admin"), bucketEntry.Extended[s3_constants.AmzIdentityId]) + assert.Equal(t, []byte("000000000000"), bucketEntry.Extended[s3_constants.ExtAmzOwnerKey]) + objectEntry := filer.entries["/buckets/"+postAuthzBucket+"/uploads/pic.png"] + require.NotNil(t, objectEntry, "the upload must land its object entry on the new bucket") + assert.NotEmpty(t, objectEntry.Chunks) + + volume.mu.Lock() + defer volume.mu.Unlock() + var stored []byte + for _, data := range volume.stored { + stored = append(stored, data...) + } + assert.Contains(t, string(stored), "payload", "the file body must reach the assigned volume") +} + +// A non-admin signer keeps the PUT semantics: the upload itself is authorized +// but the lazy create is refused, surfacing AccessDenied rather than creating +// the bucket or answering NoSuchBucket. +func TestPostPolicyBucketHandlerAutoCreateDeniedForNonAdmin(t *testing.T) { + volume := startFakeVolumeServer(t) + filer := &ambiguousPutFiler{ + volume: volume, + entries: map[string]*filer_pb.Entry{}, + apply: true, + } + s3a := newPostAutoCreateServer(t, filer) + + rec := httptest.NewRecorder() + s3a.PostPolicyBucketHandler(rec, newSignedPostRequestFor(t, "uploads/pic.png", postAuthzAccessKey, postAuthzSecretKey, nil)) + + assert.Equal(t, http.StatusForbidden, rec.Code, "body: %s", rec.Body.String()) + assert.Contains(t, rec.Body.String(), "AccessDenied") + + filer.mu.Lock() + defer filer.mu.Unlock() + assert.Empty(t, filer.entries, "a non-admin POST must not create the bucket") +}