s3: option to disable bucket auto-creation on upload (#10759)

* s3: add option to disable bucket auto-creation on upload

* command: expose -autoCreateBucket in s3, filer, server, and mini

* s3: apply the bucket auto-create policy to directory marker uploads

* s3: validate the bucket name before the auto-create disabled check

* s3: cover the disabled auto-create gate at all three upload entry points

weed/s3api/s3api_bucket_handlers.go

@@ -589,6 +589,9 @@ func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorC
 // ErrAutoCreatePermissionDenied is returned when a user lacks permission to auto-create buckets
 var ErrAutoCreatePermissionDenied = errors.New("permission denied - requires Admin permission")
 
+// ErrAutoCreateDisabled is returned when bucket auto-creation is disabled by configuration
+var ErrAutoCreateDisabled = errors.New("bucket auto-creation is disabled")
+
 // ErrInvalidBucketName is returned when a bucket name doesn't meet S3 naming requirements
 var ErrInvalidBucketName = errors.New("invalid bucket name")
 
@@ -690,6 +693,10 @@ func (s3a *S3ApiServer) autoCreateBucket(r *http.Request, bucket string) error {
 		return fmt.Errorf("auto-create bucket %s: %w", bucket, errors.Join(ErrInvalidBucketName, err))
 	}
 
+	if !s3a.option.AutoCreateBucket {
+		return fmt.Errorf("auto-create bucket %s: %w", bucket, ErrAutoCreateDisabled)
+	}
+
 	// Check if user has admin permissions
 	if !s3a.isUserAdmin(r) {
 		return fmt.Errorf("auto-create bucket %s: %w", bucket, ErrAutoCreatePermissionDenied)
@@ -758,7 +765,9 @@ func (s3a *S3ApiServer) handleAutoCreateBucket(w http.ResponseWriter, r *http.Re
 	if err := s3a.autoCreateBucket(r, bucket); err != nil {
 		glog.Warningf("%s: %v", handlerName, err)
 		// Check for specific errors to return appropriate S3 error codes
-		if errors.Is(err, ErrInvalidBucketName) {
+		if errors.Is(err, ErrAutoCreateDisabled) {
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
+		} else if errors.Is(err, ErrInvalidBucketName) {
 			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidBucketName)
 		} else if errors.Is(err, ErrAutoCreatePermissionDenied) {
 			s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)

weed/s3api/s3api_bucket_handlers_misc_test.go

@@ -221,3 +221,67 @@ func TestGetBucketLogging(t *testing.T) {
 		t.Fatalf("missing xmlns: %s", got)
 	}
 }
+
+func TestHandleAutoCreateBucketDisabled(t *testing.T) {
+	s3a := &S3ApiServer{option: &S3ApiServerOption{}}
+	req := newBucketRequest(http.MethodPut, "test-bucket", "", "")
+	rec := httptest.NewRecorder()
+
+	if s3a.handleAutoCreateBucket(rec, req, "test-bucket", "PutObjectHandler") {
+		t.Fatal("expected auto-create to be rejected")
+	}
+	if rec.Code != http.StatusNotFound {
+		t.Fatalf("status = %d, want %d", rec.Code, http.StatusNotFound)
+	}
+	if !strings.Contains(rec.Body.String(), "NoSuchBucket") {
+		t.Fatalf("body = %s, want NoSuchBucket", rec.Body.String())
+	}
+}
+
+func TestHandleAutoCreateBucketNonAdmin(t *testing.T) {
+	s3a := &S3ApiServer{option: &S3ApiServerOption{AutoCreateBucket: true}}
+	req := newBucketRequest(http.MethodPut, "test-bucket", "", "")
+	rec := httptest.NewRecorder()
+
+	if s3a.handleAutoCreateBucket(rec, req, "test-bucket", "PutObjectHandler") {
+		t.Fatal("expected auto-create to be rejected")
+	}
+	if rec.Code != http.StatusForbidden {
+		t.Fatalf("status = %d, want %d", rec.Code, http.StatusForbidden)
+	}
+}
+
+func TestUploadMissingBucketAutoCreateDisabled(t *testing.T) {
+	cases := []struct {
+		name    string
+		method  string
+		object  string
+		handler func(*S3ApiServer, http.ResponseWriter, *http.Request)
+	}{
+		{"put object", http.MethodPut, "/key", (*S3ApiServer).PutObjectHandler},
+		{"put directory marker", http.MethodPut, "/dir/", (*S3ApiServer).PutObjectHandler},
+		{"new multipart upload", http.MethodPost, "/key", (*S3ApiServer).NewMultipartUploadHandler},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			s3a := &S3ApiServer{
+				option:            &S3ApiServerOption{},
+				iam:               &IdentityAccessManagement{},
+				bucketConfigCache: NewBucketConfigCache(time.Minute),
+			}
+			s3a.bucketConfigCache.SetNegativeCache("missing")
+			req := httptest.NewRequest(tc.method, "/missing"+tc.object, strings.NewReader(""))
+			req = mux.SetURLVars(req, map[string]string{"bucket": "missing", "object": tc.object})
+			rec := httptest.NewRecorder()
+
+			tc.handler(s3a, rec, req)
+
+			if rec.Code != http.StatusNotFound {
+				t.Fatalf("status = %d, want %d", rec.Code, http.StatusNotFound)
+			}
+			if !strings.Contains(rec.Body.String(), "NoSuchBucket") {
+				t.Fatalf("body = %s, want NoSuchBucket", rec.Body.String())
+			}
+		})
+	}
+}

weed/s3api/s3api_object_handlers_put.go

@@ -131,6 +131,17 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
 
 	objectContentType := r.Header.Get("Content-Type")
 	if strings.HasSuffix(object, "/") && r.ContentLength <= 1024 {
+		// The filer mkdir below would implicitly create a missing bucket, so gate it
+		// on the same auto-create policy as regular uploads.
+		if err := s3a.checkBucket(r, bucket); err == s3err.ErrNoSuchBucket {
+			if !s3a.handleAutoCreateBucket(w, r, bucket, "PutObjectHandler") {
+				return
+			}
+		} else if err != s3err.ErrNone {
+			s3err.WriteErrorResponse(w, r, err)
+			return
+		}
+
 		// Split the object into directory path and name
 		objectWithoutSlash := strings.TrimSuffix(object, "/")
 		dirName := path.Dir(objectWithoutSlash)

weed/s3api/s3api_server.go

@@ -52,6 +52,7 @@ type S3ApiServerOption struct {
 	BucketsPath               string
 	GrpcDialOption            grpc.DialOption
 	AllowDeleteBucketNotEmpty bool
+	AutoCreateBucket          bool // create the bucket on upload if it does not exist
 	LocalFilerSocket          string
 	DataCenter                string
 	FilerGroup                string