filer: configurable TUS max upload size and session expiry (#10638)

* make TUS max upload size and session expiry configurable

* default TUS session expiry to 24h

weed/command/filer.go

@@ -80,6 +80,8 @@ type FilerOptions struct {
 	allowedOrigins            *string
 	exposeDirectoryData       *bool
 	tusBasePath               *string
+	tusMaxSizeMB              *int
+	tusSessionExpiry          *time.Duration
 	s3ConfigFile              *string // optional path to static S3 identity config
 	// shutdownCtx, when non-nil, tells startFiler to gracefully shut down its
 	// HTTP/gRPC servers once the ctx is cancelled. Used by integration tests
@@ -123,6 +125,8 @@ func init() {
 	f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins")
 	f.exposeDirectoryData = cmdFiler.Flag.Bool("exposeDirectoryData", true, "whether to return directory metadata and content in Filer UI")
 	f.tusBasePath = cmdFiler.Flag.String("tusBasePath", "/.tus", "TUS resumable upload endpoint base path (e.g., /.tus)")
+	f.tusMaxSizeMB = cmdFiler.Flag.Int("tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")
+	f.tusSessionExpiry = cmdFiler.Flag.Duration("tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration, e.g. \"48h\", \"7h30m\"")
 
 	// start s3 on filer
 	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -386,6 +390,8 @@ func (fo *FilerOptions) startFiler() {
 		DiskType:                  *fo.diskType,
 		AllowedOrigins:            strings.Split(*fo.allowedOrigins, ","),
 		TusBasePath:               *fo.tusBasePath,
+		TusMaxSize:                int64(*fo.tusMaxSizeMB) * 1024 * 1024,
+		TusSessionExpiry:          *fo.tusSessionExpiry,
 		CredentialManager:         credentialManager,
 	})
 	if nfs_err != nil {

weed/command/mini.go

@@ -458,6 +458,8 @@ func initMiniFilerFlags() {
 	miniFilerOptions.allowedOrigins = cmdMini.Flag.String("filer.allowedOrigins", "*", "comma separated list of allowed origins")
 	miniFilerOptions.exposeDirectoryData = cmdMini.Flag.Bool("filer.exposeDirectoryData", true, "whether to return directory metadata and content in Filer UI")
 	miniFilerOptions.tusBasePath = cmdMini.Flag.String("filer.tusBasePath", "/.tus", "TUS resumable upload endpoint base path")
+	miniFilerOptions.tusMaxSizeMB = cmdMini.Flag.Int("filer.tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")
+	miniFilerOptions.tusSessionExpiry = cmdMini.Flag.Duration("filer.tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration")
 }
 
 // initMiniVolumeFlags initializes Volume server flag options

weed/command/server.go

@@ -130,6 +130,8 @@ func init() {
 	filerOptions.diskType = cmdServer.Flag.String("filer.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	filerOptions.exposeDirectoryData = cmdServer.Flag.Bool("filer.exposeDirectoryData", true, "expose directory data via filer. If false, filer UI will be inaccessible.")
 	filerOptions.tusBasePath = cmdServer.Flag.String("filer.tusBasePath", "/.tus", "TUS resumable upload endpoint base path (e.g., /.tus)")
+	filerOptions.tusMaxSizeMB = cmdServer.Flag.Int("filer.tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")
+	filerOptions.tusSessionExpiry = cmdServer.Flag.Duration("filer.tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration, e.g. \"48h\", \"7h30m\"")
 
 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")

weed/server/filer_server.go

@@ -84,6 +84,8 @@ type FilerOption struct {
 	AllowedOrigins            []string
 	ExposeDirectoryData       bool
 	TusBasePath               string
+	TusMaxSize                int64
+	TusSessionExpiry          time.Duration
 	S3ConfigFile              string // optional path to static S3 identity config file
 	CredentialManager         *credential.CredentialManager
 }
@@ -254,6 +256,12 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 			if option.TusBasePath == "" {
 				glog.Warningf("Invalid TUS base path; TUS disabled (must not be root '/')")
 			} else {
+				if option.TusMaxSize <= 0 {
+					option.TusMaxSize = TusDefaultMaxSize
+				}
+				if option.TusSessionExpiry <= 0 {
+					option.TusSessionExpiry = TusDefaultSessionExpiry
+				}
 				handlePath := option.TusBasePath + "/"
 				defaultMux.HandleFunc(handlePath, fs.filerGuard.WhiteList(requestIDMiddleware(fs.tusHandler)))
 				// Start background cleanup of expired TUS sessions (every hour)

weed/server/filer_server_tus_handlers.go

@@ -175,7 +175,7 @@ func writeTusCompleteError(w http.ResponseWriter, err error) {
 func (fs *FilerServer) tusOptionsHandler(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Tus-Version", TusVersion)
 	w.Header().Set("Tus-Extension", TusExtensions)
-	w.Header().Set("Tus-Max-Size", strconv.FormatInt(TusMaxSize, 10))
+	w.Header().Set("Tus-Max-Size", strconv.FormatInt(fs.option.TusMaxSize, 10))
 	w.WriteHeader(http.StatusOK)
 }
 
@@ -197,7 +197,7 @@ func (fs *FilerServer) tusCreateHandler(w http.ResponseWriter, r *http.Request)
 		http.Error(w, "Invalid Upload-Length", http.StatusBadRequest)
 		return
 	}
-	if uploadLength > TusMaxSize {
+	if uploadLength > fs.option.TusMaxSize {
 		http.Error(w, "Upload-Length exceeds maximum", http.StatusRequestEntityTooLarge)
 		return
 	}

weed/server/filer_server_tus_idor_test.go

@@ -82,7 +82,7 @@ func TestFilerServer_tusHandler_RejectsAliasesAndInvalidMetadata(t *testing.T) {
 		{"empty stored target", tusTestUploadID, TusSession{ID: tusTestUploadID, TargetPath: "", Size: 1}},
 		{"root stored target", tusTestUploadID, TusSession{ID: tusTestUploadID, TargetPath: "/", Size: 1}},
 		{"relative stored target", tusTestUploadID, TusSession{ID: tusTestUploadID, TargetPath: "buckets/secret/x.bin", Size: 1}},
-		{"oversize stored size", tusTestUploadID, TusSession{ID: tusTestUploadID, TargetPath: "/buckets/secret/victim.bin", Size: TusMaxSize + 1}},
+		{"oversize stored size", tusTestUploadID, TusSession{ID: tusTestUploadID, TargetPath: "/buckets/secret/victim.bin", Size: TusDefaultMaxSize + 1}},
 	}
 
 	for _, tt := range tests {

weed/server/filer_server_tus_jwt_test.go

@@ -48,7 +48,7 @@ func newTusTestServer(t *testing.T, sessions map[string]string) (*FilerServer, *
 	fs := &FilerServer{
 		filer:      newRenameTestFiler(t, store),
 		filerGuard: security.NewGuard(nil, tusTestWriteKey, 0, tusTestReadKey, 0),
-		option:     &FilerOption{TusBasePath: "/.tus"},
+		option:     &FilerOption{TusBasePath: "/.tus", TusMaxSize: TusDefaultMaxSize},
 	}
 	for uploadID, targetPath := range sessions {
 		seedTusSession(t, fs, store, TusSession{ID: uploadID, TargetPath: targetPath, Size: 1})

weed/server/filer_server_tus_session.go

@@ -20,12 +20,13 @@ import (
 )
 
 const (
-	TusVersion       = "1.0.0"
+	TusVersion              = "1.0.0"
-	TusMaxSize       = int64(5 * 1024 * 1024 * 1024) // 5GB default max size
+	TusDefaultMaxSize       = int64(5 * 1024 * 1024 * 1024) // 5GB
-	TusUploadsFolder = ".uploads.tus"
+	TusDefaultSessionExpiry = 24 * time.Hour
-	TusInfoFileName  = ".info"
+	TusUploadsFolder        = ".uploads.tus"
-	TusChunkExt      = ".chunk"
+	TusInfoFileName         = ".info"
-	TusExtensions    = "creation,creation-with-upload,termination"
+	TusChunkExt             = ".chunk"
+	TusExtensions           = "creation,creation-with-upload,termination"
 )
 
 // ErrWormEnforced marks a TUS completion rejected because the target entry is
@@ -124,7 +125,7 @@ func (fs *FilerServer) createTusSession(ctx context.Context, uploadID, targetPat
 		Offset:     0,
 		Metadata:   metadata,
 		CreatedAt:  time.Now(),
-		ExpiresAt:  time.Now().Add(7 * 24 * time.Hour), // 7 days default expiration
+		ExpiresAt:  time.Now().Add(fs.option.TusSessionExpiry),
 		Chunks:     []*TusChunkInfo{},
 	}
 
@@ -209,7 +210,7 @@ func (fs *FilerServer) readTusSessionInfo(ctx context.Context, uploadID string)
 	if target == "" || target == "/" {
 		return nil, fmt.Errorf("invalid TUS target path: %q", session.TargetPath)
 	}
-	if session.Size < 0 || session.Size > TusMaxSize {
+	if session.Size < 0 || session.Size > fs.option.TusMaxSize {
 		return nil, fmt.Errorf("invalid TUS upload size: %d", session.Size)
 	}
 	// Pin authorization and every later operation to the same canonical path.