s3: chunk uploads at the filer's maxMB (#10439)

The S3 write path cut fixed 8MB chunks, so an object stored through S3
chunked differently from the same bytes stored through the filer, WebDAV
or a mount, and -maxMB had no effect on it. Read maxMB from the filer
configuration at startup and use it, falling back to 8MB when the filer
reports none.

weed/command/s3.go

@@ -277,6 +277,9 @@ func (s3opt *S3Options) startS3Server() bool {
 	var metricsAddress string
 	var metricsIntervalSec int
 
+	// chunk size for S3 uploads, read from the filer's -maxMB
+	var filerMaxMB int32
+
 	for {
 		err := pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
@@ -288,7 +291,9 @@ func (s3opt *S3Options) startS3Server() bool {
 			// Get master addresses for filer discovery
 			masterAddresses = pb.ServerAddresses(strings.Join(resp.Masters, ",")).ToAddresses()
 			metricsAddress, metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSec)
+			filerMaxMB = int32(resp.MaxMb)
 			glog.V(0).Infof("S3 read filer buckets dir: %s", filerBucketsPath)
+			glog.V(0).Infof("S3 read filer maxMB: %d", filerMaxMB)
 			if len(masterAddresses) > 0 {
 				glog.V(0).Infof("S3 read master addresses for discovery: %v", masterAddresses)
 			}
@@ -358,6 +363,7 @@ func (s3opt *S3Options) startS3Server() bool {
 		ExternalUrl:               s3opt.resolveExternalUrl(),
 		DefaultFileMode:           defaultFileMode,
 		CacheSizeMB:               *s3opt.cacheSizeMB,
+		MaxMB:                     filerMaxMB,
 	})
 	if s3ApiServer_err != nil {
 		glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)

weed/s3api/s3api_object_handlers_put.go

@@ -363,6 +363,27 @@ type putFinalize struct {
 	afterCreate func(entry *filer_pb.Entry) s3err.ErrorCode
 }
 
+const (
+	// defaultUploadChunkSizeMB applies when the filer reports no -maxMB.
+	defaultUploadChunkSizeMB = 8
+	// maxUploadChunkSizeMB keeps the byte count inside int32.
+	maxUploadChunkSizeMB = 2047
+)
+
+// uploadChunkSize is how large a chunk the S3 write path cuts. It follows the
+// filer's -maxMB so an object written through S3 chunks the same way the same
+// bytes written through the filer, WebDAV or a mount would.
+func (s3a *S3ApiServer) uploadChunkSize() int32 {
+	sizeMB := s3a.option.MaxMB
+	if sizeMB <= 0 {
+		sizeMB = defaultUploadChunkSizeMB
+	}
+	if sizeMB > maxUploadChunkSizeMB {
+		sizeMB = maxUploadChunkSizeMB
+	}
+	return sizeMB * 1024 * 1024
+}
+
 // putToFiler writes one chunk of object bytes (a full PutObject body, a
 // single MPU part, a copy-part destination). lifecycleTTLSec is non-zero
 // only for top-level PutObject paths where the lifecycle XML's
@@ -466,8 +487,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
 
 	// filePath is already provided directly - no URL parsing needed
 	// Step 1 & 2: Use auto-chunking to handle large files without OOM
-	// This splits large uploads into 8MB chunks, preventing memory issues on both S3 API and volume servers
-	const chunkSize = 8 * 1024 * 1024 // 8MB chunks (S3 standard)
+	chunkSize := s3a.uploadChunkSize()
 	const smallFileLimit = 256 * 1024 // 256KB - store inline in filer
 
 	collection := ""

weed/s3api/s3api_object_handlers_put_chunksize_test.go

@@ -0,0 +1,26 @@
+package s3api
+
+import "testing"
+
+func TestUploadChunkSizeFollowsFilerMaxMB(t *testing.T) {
+	cases := []struct {
+		name     string
+		maxMB    int32
+		expected int32
+	}{
+		{"filer maxMB unset", 0, defaultUploadChunkSizeMB * 1024 * 1024},
+		{"filer maxMB negative", -1, defaultUploadChunkSizeMB * 1024 * 1024},
+		{"filer default", 4, 4 * 1024 * 1024},
+		{"filer maxMB 32", 32, 32 * 1024 * 1024},
+		{"clamped to int32 range", 4096, maxUploadChunkSizeMB * 1024 * 1024},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			s3a := &S3ApiServer{option: &S3ApiServerOption{MaxMB: c.maxMB}}
+			if got := s3a.uploadChunkSize(); got != c.expected {
+				t.Errorf("uploadChunkSize() = %d, want %d", got, c.expected)
+			}
+		})
+	}
+}

weed/s3api/s3api_server.go

@@ -66,6 +66,7 @@ type S3ApiServerOption struct {
 	ExternalUrl               string // external URL clients use, for signature verification behind a reverse proxy
 	DefaultFileMode           uint32 // default file permission mode for S3 uploads (e.g. 0660, 0644)
 	CacheSizeMB               int64  // in-memory chunk cache capacity in MB for the shared ReaderCache; 0 disables
+	MaxMB                     int32  // filer's -maxMB, read from the filer configuration at startup
 }
 
 // s3ChunkCacheChunkSizeMB is the assumed chunk size (in MiB) used to convert