diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.go index eadd89b3c9..fd9abe2e0d 100644 --- a/weed/admin/dash/admin_server.go +++ b/weed/admin/dash/admin_server.go @@ -175,8 +175,14 @@ type AdminServer struct { // s3PublicEndpoint is the client-facing S3 address from admin.toml // (s3.public_endpoint); discovered S3 servers are the fallback. s3PublicEndpoint string + + // uploadMaxSize caps the total bytes of one file browser upload; 0 means no limit. + uploadMaxSize int64 } +// DefaultUploadMaxSizeMB is the default total size limit of one file browser upload. +const DefaultUploadMaxSizeMB = 500 + // Type definitions moved to types.go func NewAdminServer(masters string, filerGroup string, templateFS http.FileSystem, dataDir string, icebergPort, lancePort int, s3PublicEndpoint string) *AdminServer { @@ -217,6 +223,7 @@ func NewAdminServer(masters string, filerGroup string, templateFS http.FileSyste icebergPort: icebergPort, lancePort: lancePort, s3PublicEndpoint: normalizeS3PublicEndpoint(s3PublicEndpoint), + uploadMaxSize: DefaultUploadMaxSizeMB << 20, pluginLock: lockManager, adminPresenceLock: presenceLock, bgCancel: bgCancel, @@ -1574,6 +1581,18 @@ func (s *AdminServer) GetS3Endpoint() string { return s.s3PublicEndpoint } +// SetUploadMaxSize sets the total size limit in bytes of one file browser +// upload; zero or less removes the limit. +func (s *AdminServer) SetUploadMaxSize(maxBytes int64) { + s.uploadMaxSize = max(maxBytes, 0) +} + +// GetUploadMaxSize returns the total size limit in bytes of one file browser +// upload, or 0 when uploads are unlimited. +func (s *AdminServer) GetUploadMaxSize() int64 { + return s.uploadMaxSize +} + // normalizeS3PublicEndpoint trims a trailing slash and drops, with a warning, // a value that is not a plain absolute http or https URL, so the file browser // hides its URL actions instead of copying broken links. Checking the string diff --git a/weed/admin/dash/file_browser_data.go b/weed/admin/dash/file_browser_data.go index 37385f5fb4..7fd14a3e6b 100644 --- a/weed/admin/dash/file_browser_data.go +++ b/weed/admin/dash/file_browser_data.go @@ -53,6 +53,8 @@ type FileBrowserData struct { HasNextPage bool `json:"has_next_page"` LastFileName string `json:"last_file_name"` // Cursor for next page CurrentLastFileName string `json:"current_last_file_name"` // Cursor from current request (for page size changes) + + UploadMaxSize int64 `json:"upload_max_size"` } // GetFileBrowser retrieves file browser data for a given path with cursor-based @@ -240,6 +242,8 @@ func (s *AdminServer) GetFileBrowser(dir string, prefix string, lastFileName str HasNextPage: hasNextPage, LastFileName: lastEntryName, // Store for next page navigation CurrentLastFileName: lastFileName, // Store input cursor for page size changes + + UploadMaxSize: s.uploadMaxSize, }, nil } diff --git a/weed/admin/handlers/file_browser_handlers.go b/weed/admin/handlers/file_browser_handlers.go index 085679b768..fb5f122b72 100644 --- a/weed/admin/handlers/file_browser_handlers.go +++ b/weed/admin/handlers/file_browser_handlers.go @@ -4,6 +4,7 @@ import ( "bufio" "compress/gzip" "context" + "errors" "fmt" "io" "mime" @@ -247,25 +248,46 @@ func (h *FileBrowserHandlers) CreateFolder(w http.ResponseWriter, r *http.Reques // UploadFile handles file upload requests func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request) { - // Get the current path + maxSize := h.adminServer.GetUploadMaxSize() + if maxSize > 0 { + r.Body = http.MaxBytesReader(w, r.Body, maxSize+uploadFormOverhead) + } + + // Parts beyond 32MB spill to temporary files instead of memory + err := r.ParseMultipartForm(32 << 20) + if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + writeJSONError(w, http.StatusRequestEntityTooLarge, uploadTooLargeMessage(maxSize)) + return + } + writeJSONError(w, http.StatusBadRequest, "Failed to parse multipart form: "+err.Error()) + return + } + // The router hands us a copy of the request, so the server cannot remove the spooled parts itself + defer r.MultipartForm.RemoveAll() + currentPath := r.FormValue("path") if currentPath == "" { currentPath = "/" } - // Parse multipart form - err := r.ParseMultipartForm(1 << 30) // 1GB max memory for large file uploads - if err != nil { - writeJSONError(w, http.StatusBadRequest, "Failed to parse multipart form: "+err.Error()) - return - } - // Get uploaded files (supports multiple files) files := r.MultipartForm.File["files"] if len(files) == 0 { writeJSONError(w, http.StatusBadRequest, "No files uploaded") return } + if maxSize > 0 { + var totalSize int64 + for _, fileHeader := range files { + totalSize += fileHeader.Size + } + if totalSize > maxSize { + writeJSONError(w, http.StatusRequestEntityTooLarge, uploadTooLargeMessage(maxSize)) + return + } + } var uploadResults []map[string]interface{} var failedUploads []string @@ -327,6 +349,15 @@ func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request) } } +// uploadFormOverhead allows for the multipart headers and the path field on +// top of the file bytes an upload may carry. Go's multipart parser accepts at +// most 1000 parts, whose headers take about half of this. +const uploadFormOverhead = 1 << 20 + +func uploadTooLargeMessage(maxSize int64) string { + return fmt.Sprintf("Total upload size exceeds the %d MB limit", maxSize>>20) +} + // uploadFileToFiler uploads a file to the cluster via filer gRPC + volume // HTTP. This works whether or not the filer is running with -disableHttp=true, // since the bytes never traverse the filer's HTTP listener. The multipart diff --git a/weed/admin/handlers/file_browser_handlers_test.go b/weed/admin/handlers/file_browser_handlers_test.go index f02fead273..179be2a6a0 100644 --- a/weed/admin/handlers/file_browser_handlers_test.go +++ b/weed/admin/handlers/file_browser_handlers_test.go @@ -1,8 +1,12 @@ package handlers import ( + "bytes" + "fmt" + "mime/multipart" "net/http" "net/http/httptest" + "os" "slices" "strings" "testing" @@ -143,3 +147,72 @@ func TestListFolders_RejectsPathOutsideBuckets(t *testing.T) { } } } + +func TestUploadFile_RejectsOversizedUpload(t *testing.T) { + adminServer := &dash.AdminServer{} + adminServer.SetUploadMaxSize(1 << 20) + h := &FileBrowserHandlers{adminServer: adminServer} + + // 1.5MB and two 600KB files fit the body allowance but exceed the file total; + // 3MB trips the body cap. + for _, sizes := range [][]int{{3 << 19}, {600 << 10, 600 << 10}, {3 << 20}} { + var body bytes.Buffer + mw := multipart.NewWriter(&body) + for i, size := range sizes { + part, err := mw.CreateFormFile("files", fmt.Sprintf("big%d.bin", i)) + if err != nil { + t.Fatal(err) + } + part.Write(make([]byte, size)) + } + mw.WriteField("path", "/buckets/b") + mw.Close() + + req := httptest.NewRequest(http.MethodPost, "/api/files/upload", &body) + req.Header.Set("Content-Type", mw.FormDataContentType()) + w := httptest.NewRecorder() + + h.UploadFile(w, req) + + if w.Code != http.StatusRequestEntityTooLarge { + t.Errorf("sizes %v: expected 413, got %d (body: %s)", sizes, w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "1 MB limit") { + t.Errorf("sizes %v: expected the limit in the error, got %s", sizes, w.Body.String()) + } + } +} + +func TestUploadFile_RemovesSpooledParts(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv("TMPDIR", tmpDir) + + adminServer := &dash.AdminServer{} + adminServer.SetUploadMaxSize(34 << 20) + h := &FileBrowserHandlers{adminServer: adminServer} + + // The 33MB part spills to disk; together the parts exceed the limit, so the upload stops there. + var body bytes.Buffer + mw := multipart.NewWriter(&body) + for i, size := range []int{33 << 20, 3 << 19} { + part, err := mw.CreateFormFile("files", fmt.Sprintf("part%d.bin", i)) + if err != nil { + t.Fatal(err) + } + part.Write(make([]byte, size)) + } + mw.Close() + + req := httptest.NewRequest(http.MethodPost, "/api/files/upload", &body) + req.Header.Set("Content-Type", mw.FormDataContentType()) + w := httptest.NewRecorder() + + h.UploadFile(w, req.WithContext(req.Context())) + + if w.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("expected 413, got %d (body: %s)", w.Code, w.Body.String()) + } + if left, _ := os.ReadDir(tmpDir); len(left) != 0 { + t.Errorf("spooled parts left behind: %v", left) + } +} diff --git a/weed/admin/static/js/admin.js b/weed/admin/static/js/admin.js index 5de9b8bd64..77681f26fc 100644 --- a/weed/admin/static/js/admin.js +++ b/weed/admin/static/js/admin.js @@ -1180,10 +1180,9 @@ async function submitUploadFile() { const files = Array.from(fileInput.files); const totalSize = files.reduce((sum, file) => sum + file.size, 0); - // Validate total file size (limit to 500MB for admin interface) - const maxSize = 500 * 1024 * 1024; // 500MB total - if (totalSize > maxSize) { - showErrorMessage('Total file size exceeds 500MB limit. Please select fewer or smaller files.'); + const maxSize = Number(document.getElementById('uploadFileForm').dataset.maxSize) || 0; + if (maxSize > 0 && totalSize > maxSize) { + showErrorMessage(`Total file size exceeds the ${formatBytes(maxSize)} limit. Please select fewer or smaller files.`); return; } diff --git a/weed/admin/static_gz/js/admin.js.gz b/weed/admin/static_gz/js/admin.js.gz index a4ebc6fd53..d6a6220718 100644 Binary files a/weed/admin/static_gz/js/admin.js.gz and b/weed/admin/static_gz/js/admin.js.gz differ diff --git a/weed/admin/view/app/file_browser.templ b/weed/admin/view/app/file_browser.templ index 4367d74698..7dc84c9069 100644 --- a/weed/admin/view/app/file_browser.templ +++ b/weed/admin/view/app/file_browser.templ @@ -310,12 +310,15 @@ templ FileBrowser(data dash.FileBrowserData) {
-
+
Choose one or more files to upload to the current directory. You can select multiple files by holding Ctrl (Cmd on Mac) while clicking. + if data.UploadMaxSize > 0 { + Total upload size limit: { formatBytes(data.UploadMaxSize) }. + }
diff --git a/weed/admin/view/app/file_browser_templ.go b/weed/admin/view/app/file_browser_templ.go index 0554245049..f994bdd27a 100644 --- a/weed/admin/view/app/file_browser_templ.go +++ b/weed/admin/view/app/file_browser_templ.go @@ -852,20 +852,56 @@ func FileBrowser(data dash.FileBrowserData) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 93, "\">
Upload Files
Choose one or more files to upload to the current directory. You can select multiple files by holding Ctrl (Cmd on Mac) while clicking.
Upload Files
0%
Preparing upload...
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, "\">
Choose one or more files to upload to the current directory. You can select multiple files by holding Ctrl (Cmd on Mac) while clicking. ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if data.UploadMaxSize > 0 { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 95, "Total upload size limit: ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var41 string + templ_7745c5c3_Var41, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(data.UploadMaxSize)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 320, Col: 67} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var41)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, ".") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 97, "
0%
Preparing upload...
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/command/admin.go b/weed/command/admin.go index a4844f1b8f..59ac89bdb0 100644 --- a/weed/command/admin.go +++ b/weed/command/admin.go @@ -64,6 +64,7 @@ type AdminOptions struct { debugPort *int cpuProfile *string memProfile *string + uploadMaxSizeMB *int // workerGrpcListener, when set, is a listener already bound to grpcPort by // the caller. `weed mini` reserves the port this way because the admin @@ -125,6 +126,7 @@ func init() { a.debugPort = cmdAdmin.Flag.Int("debug.port", 6060, "http port for debugging") a.cpuProfile = cmdAdmin.Flag.String("cpuprofile", "", "cpu profile output file") a.memProfile = cmdAdmin.Flag.String("memprofile", "", "memory profile output file") + a.uploadMaxSizeMB = cmdAdmin.Flag.Int("uploadMaxSizeMB", dash.DefaultUploadMaxSizeMB, "maximum total size in MB of one file browser upload (0 for no limit)") } var cmdAdmin = &Command{ @@ -500,6 +502,9 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool, s3PublicEndpoint = options.defaultS3PublicEndpoint } adminServer := dash.NewAdminServer(*options.master, *options.filerGroup, nil, dataDir, icebergPort, lancePort, s3PublicEndpoint) + if options.uploadMaxSizeMB != nil { + adminServer.SetUploadMaxSize(int64(*options.uploadMaxSizeMB) << 20) + } if err := adminServer.ApplyPluginConfigFromToml(util.GetViper()); err != nil { return fmt.Errorf("apply admin.toml to plugin config: %w", err) diff --git a/weed/command/mini.go b/weed/command/mini.go index 498635d302..2d963f7bda 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -17,6 +17,7 @@ import ( "sync" "time" + "github.com/seaweedfs/seaweedfs/weed/admin/dash" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -598,6 +599,7 @@ func initMiniAdminFlags() { miniAdminOptions.allowInsecureBind = cmdMini.Flag.Bool("admin.allowInsecureBind", false, "INSECURE: allow the unauthenticated admin interface to bind to -ip.bind instead of loopback") miniAdminWorkerBindIP = cmdMini.Flag.String("admin.worker.ip", "127.0.0.1", "worker gRPC bind address; expose only with grpc.admin mTLS configured") miniAdminOptions.urlPrefix = cmdMini.Flag.String("admin.urlPrefix", "", "URL path prefix when running the admin UI behind a reverse proxy under a subdirectory (e.g. /seaweedfs)") + miniAdminOptions.uploadMaxSizeMB = cmdMini.Flag.Int("admin.uploadMaxSizeMB", dash.DefaultUploadMaxSizeMB, "maximum total size in MB of one file browser upload (0 for no limit)") } func init() {