diff --git a/weed/command/s3.go b/weed/command/s3.go index 1cb0b87807..1ed958b00f 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -277,6 +277,9 @@ func (s3opt *S3Options) startS3Server() bool { var metricsAddress string var metricsIntervalSec int + // chunk size for S3 uploads, read from the filer's -maxMB + var filerMaxMB int32 + for { err := pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) @@ -288,7 +291,9 @@ func (s3opt *S3Options) startS3Server() bool { // Get master addresses for filer discovery masterAddresses = pb.ServerAddresses(strings.Join(resp.Masters, ",")).ToAddresses() metricsAddress, metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSec) + filerMaxMB = int32(resp.MaxMb) glog.V(0).Infof("S3 read filer buckets dir: %s", filerBucketsPath) + glog.V(0).Infof("S3 read filer maxMB: %d", filerMaxMB) if len(masterAddresses) > 0 { glog.V(0).Infof("S3 read master addresses for discovery: %v", masterAddresses) } @@ -358,6 +363,7 @@ func (s3opt *S3Options) startS3Server() bool { ExternalUrl: s3opt.resolveExternalUrl(), DefaultFileMode: defaultFileMode, CacheSizeMB: *s3opt.cacheSizeMB, + MaxMB: filerMaxMB, }) if s3ApiServer_err != nil { glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err) diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index fa6fdcb96a..1f01a3ed37 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -363,6 +363,27 @@ type putFinalize struct { afterCreate func(entry *filer_pb.Entry) s3err.ErrorCode } +const ( + // defaultUploadChunkSizeMB applies when the filer reports no -maxMB. + defaultUploadChunkSizeMB = 8 + // maxUploadChunkSizeMB keeps the byte count inside int32. + maxUploadChunkSizeMB = 2047 +) + +// uploadChunkSize is how large a chunk the S3 write path cuts. It follows the +// filer's -maxMB so an object written through S3 chunks the same way the same +// bytes written through the filer, WebDAV or a mount would. +func (s3a *S3ApiServer) uploadChunkSize() int32 { + sizeMB := s3a.option.MaxMB + if sizeMB <= 0 { + sizeMB = defaultUploadChunkSizeMB + } + if sizeMB > maxUploadChunkSizeMB { + sizeMB = maxUploadChunkSizeMB + } + return sizeMB * 1024 * 1024 +} + // putToFiler writes one chunk of object bytes (a full PutObject body, a // single MPU part, a copy-part destination). lifecycleTTLSec is non-zero // only for top-level PutObject paths where the lifecycle XML's @@ -466,8 +487,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader // filePath is already provided directly - no URL parsing needed // Step 1 & 2: Use auto-chunking to handle large files without OOM - // This splits large uploads into 8MB chunks, preventing memory issues on both S3 API and volume servers - const chunkSize = 8 * 1024 * 1024 // 8MB chunks (S3 standard) + chunkSize := s3a.uploadChunkSize() const smallFileLimit = 256 * 1024 // 256KB - store inline in filer collection := "" diff --git a/weed/s3api/s3api_object_handlers_put_chunksize_test.go b/weed/s3api/s3api_object_handlers_put_chunksize_test.go new file mode 100644 index 0000000000..6269654a16 --- /dev/null +++ b/weed/s3api/s3api_object_handlers_put_chunksize_test.go @@ -0,0 +1,26 @@ +package s3api + +import "testing" + +func TestUploadChunkSizeFollowsFilerMaxMB(t *testing.T) { + cases := []struct { + name string + maxMB int32 + expected int32 + }{ + {"filer maxMB unset", 0, defaultUploadChunkSizeMB * 1024 * 1024}, + {"filer maxMB negative", -1, defaultUploadChunkSizeMB * 1024 * 1024}, + {"filer default", 4, 4 * 1024 * 1024}, + {"filer maxMB 32", 32, 32 * 1024 * 1024}, + {"clamped to int32 range", 4096, maxUploadChunkSizeMB * 1024 * 1024}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + s3a := &S3ApiServer{option: &S3ApiServerOption{MaxMB: c.maxMB}} + if got := s3a.uploadChunkSize(); got != c.expected { + t.Errorf("uploadChunkSize() = %d, want %d", got, c.expected) + } + }) + } +} diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 5943bfecaa..413984cd04 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -66,6 +66,7 @@ type S3ApiServerOption struct { ExternalUrl string // external URL clients use, for signature verification behind a reverse proxy DefaultFileMode uint32 // default file permission mode for S3 uploads (e.g. 0660, 0644) CacheSizeMB int64 // in-memory chunk cache capacity in MB for the shared ReaderCache; 0 disables + MaxMB int32 // filer's -maxMB, read from the filer configuration at startup } // s3ChunkCacheChunkSizeMB is the assumed chunk size (in MiB) used to convert