diff --git a/weed/admin/dash/client_management.go b/weed/admin/dash/client_management.go index 974d996fcf..009b08ce5a 100644 --- a/weed/admin/dash/client_management.go +++ b/weed/admin/dash/client_management.go @@ -12,6 +12,8 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/wdclient" + "google.golang.org/grpc" ) // WithMasterClient executes a function with a master client connection @@ -38,6 +40,21 @@ func (s *AdminServer) WithVolumeServerClient(address pb.ServerAddress, f func(cl }) } +// GetMasterClient returns the admin server's wdclient.MasterClient. It is used +// by file browser download paths that stream chunks straight from the volume +// servers via filer.PrepareStreamContent so they keep working when the filer +// has -disableHttp=true. +func (s *AdminServer) GetMasterClient() *wdclient.MasterClient { + return s.masterClient +} + +// GetGrpcDialOption returns the dial option used for all admin-originated +// gRPC connections (TLS or insecure). File browser uploads need this when +// they perform the assign + volume HTTP POST + create-entry flow. +func (s *AdminServer) GetGrpcDialOption() grpc.DialOption { + return s.grpcDialOption +} + // GetFilerAddress returns a filer address, discovering from masters if needed func (s *AdminServer) GetFilerAddress() string { // Discover filers from masters diff --git a/weed/admin/handlers/file_browser_grpc.go b/weed/admin/handlers/file_browser_grpc.go new file mode 100644 index 0000000000..9baf51d100 --- /dev/null +++ b/weed/admin/handlers/file_browser_grpc.go @@ -0,0 +1,291 @@ +package handlers + +import ( + "bytes" + "context" + "fmt" + "io" + "mime" + "net/http" + "path" + "strconv" + "time" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// Admin file browser upload chunk sizing — kept in sync with the values +// s3api uses so files end up split into the same fid-sized pieces the rest of +// the cluster expects. +const ( + adminUploadChunkSize = 8 * 1024 * 1024 + adminUploadSmallFileLimit = 256 * 1024 +) + +// File browser handlers backed by the filer gRPC service. They bypass the +// filer's HTTP listener so the UI keeps working when the filer is started +// with -disableHttp=true; chunk bytes still flow through the volume server +// HTTP endpoints (which run on their own ports). + +// fetchFileContentGrpc reads file content via the filer gRPC service, looking +// the entry up and then streaming the chunks straight from the volume servers. +// When maxBytes > 0 the stream is truncated to that many bytes — used by the +// "is this text?" sniff so unknown-MIME files don't get fully downloaded. +func (h *FileBrowserHandlers) fetchFileContentGrpc(ctx context.Context, filePath string, maxBytes int) (string, error) { + cleanFilePath, err := h.validateAndCleanFilePath(filePath) + if err != nil { + return "", err + } + + entry, err := h.lookupEntry(ctx, cleanFilePath) + if err != nil { + return "", err + } + if entry.IsDirectory { + return "", fmt.Errorf("path is a directory") + } + + size := int64(filer.FileSize(entry)) + streamSize := size + if maxBytes > 0 && streamSize > int64(maxBytes) { + streamSize = int64(maxBytes) + } + + var buf bytes.Buffer + if err := h.streamEntryContent(ctx, entry, streamSize, &buf); err != nil { + return "", err + } + return buf.String(), nil +} + +// downloadFileGrpc streams a file via gRPC + volume server HTTP. The +// response writer receives the canonical attachment headers and the raw +// bytes; this replaces the HTTP-to-filer proxy that used to run in +// DownloadFile. +func (h *FileBrowserHandlers) downloadFileGrpc(ctx context.Context, filePath string, w http.ResponseWriter) error { + cleanFilePath, err := h.validateAndCleanFilePath(filePath) + if err != nil { + return err + } + + entry, err := h.lookupEntry(ctx, cleanFilePath) + if err != nil { + return err + } + if entry.IsDirectory { + return fmt.Errorf("path is a directory") + } + + size := int64(filer.FileSize(entry)) + + fileName := path.Base(cleanFilePath) + w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": fileName})) + + contentType := "" + if entry.Attributes != nil { + contentType = entry.Attributes.Mime + } + if contentType == "" { + contentType = "application/octet-stream" + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + w.WriteHeader(http.StatusOK) + + return h.streamEntryContent(ctx, entry, size, w) +} + +// uploadFileGrpc streams the upload to volume servers in 8 MiB chunks via the +// shared chunked-upload helper, then registers the assembled entry through the +// filer gRPC service. Bytes never enter the admin process's heap as a whole — +// each chunk is sized to adminUploadChunkSize. Small files (< 256 KiB) are +// stored inline on the entry, matching the S3 server's behaviour. +func (h *FileBrowserHandlers) uploadFileGrpc(ctx context.Context, filePath string, fileName string, mimeType string, reader io.Reader) error { + cleanFilePath, err := h.validateAndCleanFilePath(filePath) + if err != nil { + return err + } + dir := path.Dir(cleanFilePath) + if dir == "." { + dir = "/" + } + entryName := path.Base(cleanFilePath) + if mimeType == "" { + mimeType = "application/octet-stream" + } + + assignFunc := func(ctx context.Context, count int, expectedDataSize uint64) (*operation.VolumeAssignRequest, *operation.AssignResult, error) { + var assignResp *filer_pb.AssignVolumeResponse + err := h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + resp, assignErr := client.AssignVolume(ctx, &filer_pb.AssignVolumeRequest{ + Count: int32(count), + Path: cleanFilePath, + ExpectedDataSize: expectedDataSize, + }) + if assignErr != nil { + return assignErr + } + if resp.Error != "" { + return fmt.Errorf("%s", resp.Error) + } + assignResp = resp + return nil + }) + if err != nil { + return nil, nil, err + } + if assignResp.Location == nil || assignResp.FileId == "" { + return nil, nil, fmt.Errorf("assign volume returned empty location") + } + return nil, &operation.AssignResult{ + Fid: assignResp.FileId, + Url: assignResp.Location.Url, + PublicUrl: assignResp.Location.PublicUrl, + Count: uint64(count), + Auth: security.EncodedJwt(assignResp.Auth), + }, nil + } + + chunkResult, err := operation.UploadReaderInChunks(ctx, reader, &operation.ChunkedUploadOption{ + ChunkSize: adminUploadChunkSize, + SmallFileLimit: adminUploadSmallFileLimit, + SaveSmallInline: true, + MimeType: mimeType, + AssignFunc: assignFunc, + }) + if err != nil { + // Partial chunks come back even on error so we can clean them up rather + // than leaving orphaned data on volume servers. + if chunkResult != nil && len(chunkResult.FileChunks) > 0 { + h.deleteOrphanedChunks(chunkResult.FileChunks) + } + return fmt.Errorf("upload: %w", err) + } + + now := time.Now() + entry := &filer_pb.Entry{ + Name: entryName, + Attributes: &filer_pb.FuseAttributes{ + FileSize: uint64(chunkResult.TotalSize), + Mtime: now.Unix(), + Crtime: now.Unix(), + FileMode: 0644, + Mime: mimeType, + }, + } + if len(chunkResult.SmallContent) > 0 { + entry.Content = chunkResult.SmallContent + } else { + entry.Chunks = chunkResult.FileChunks + } + + err = h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + _, createErr := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{ + Directory: dir, + Entry: entry, + }) + return createErr + }) + if err != nil { + if len(chunkResult.FileChunks) > 0 { + h.deleteOrphanedChunks(chunkResult.FileChunks) + } + return fmt.Errorf("create entry: %w", err) + } + return nil +} + +// deleteOrphanedChunks best-effort removes the chunk fids when an upload +// fails partway through. Errors are logged; we can't surface them past the +// caller's primary failure. +func (h *FileBrowserHandlers) deleteOrphanedChunks(chunks []*filer_pb.FileChunk) { + fileIds := make([]string, 0, len(chunks)) + for _, c := range chunks { + if fid := c.GetFileIdString(); fid != "" { + fileIds = append(fileIds, fid) + } + } + if len(fileIds) == 0 { + return + } + master := h.adminServer.GetMasterClient() + results := operation.DeleteFileIds(master.GetMaster, false, h.adminServer.GetGrpcDialOption(), fileIds) + for _, r := range results { + if r.Error != "" { + glog.Warningf("admin file browser: orphan chunk %s cleanup: %s", r.FileId, r.Error) + } + } +} + +func (h *FileBrowserHandlers) lookupEntry(ctx context.Context, cleanFilePath string) (*filer_pb.Entry, error) { + dir := path.Dir(cleanFilePath) + if dir == "." { + dir = "/" + } + name := path.Base(cleanFilePath) + var entry *filer_pb.Entry + err := h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + resp, lookupErr := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{ + Directory: dir, + Name: name, + }) + if lookupErr != nil { + return lookupErr + } + if resp.Entry == nil { + return fmt.Errorf("not found") + } + entry = resp.Entry + return nil + }) + return entry, err +} + +func (h *FileBrowserHandlers) streamEntryContent(ctx context.Context, entry *filer_pb.Entry, size int64, w io.Writer) error { + if size == 0 { + // Inline content (small files stored directly on the entry) skip the + // chunk pipeline entirely. + if len(entry.Content) > 0 { + _, err := w.Write(entry.Content) + return err + } + return nil + } + if len(entry.Content) > 0 && len(entry.GetChunks()) == 0 { + _, err := w.Write(entry.Content) + return err + } + + streamFn, err := filer.PrepareStreamContentWithThrottler( + ctx, + h.adminServer.GetMasterClient(), + volumeServerReadJwt, + entry.GetChunks(), + 0, + size, + 0, + ) + if err != nil { + return fmt.Errorf("prepare stream: %w", err) + } + return streamFn(w) +} + +// volumeServerReadJwt mints a per-fileId Bearer token for reads against a +// volume server when jwt.signing.read.key is configured. The volume servers +// are unaware of jwt.filer_signing.read.key — that one only gates the filer +// HTTP surface, which this code path doesn't touch. +func volumeServerReadJwt(fileId string) string { + v := util.GetViper() + signingKey := security.SigningKey(v.GetString("jwt.signing.read.key")) + if len(signingKey) == 0 { + return "" + } + expiresAfterSec := v.GetInt("jwt.signing.read.expires_after_seconds") + return string(security.GenJwtForVolumeServer(signingKey, expiresAfterSec, fileId)) +} diff --git a/weed/admin/handlers/file_browser_handlers.go b/weed/admin/handlers/file_browser_handlers.go index 0f8cea0d9e..419be4b1d8 100644 --- a/weed/admin/handlers/file_browser_handlers.go +++ b/weed/admin/handlers/file_browser_handlers.go @@ -1,15 +1,10 @@ package handlers import ( - "bytes" "context" "fmt" - "io" - "mime" "mime/multipart" - "net" "net/http" - "net/url" "os" "path" "path/filepath" @@ -21,9 +16,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/admin/view/app" "github.com/seaweedfs/seaweedfs/weed/admin/view/layout" "github.com/seaweedfs/seaweedfs/weed/glog" - "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" - "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util/http/client" ) @@ -49,15 +42,6 @@ func NewFileBrowserHandlers(adminServer *dash.AdminServer) *FileBrowserHandlers } } -// newClientWithTimeout creates a temporary http.Client with the specified timeout, -// reusing the TLS transport from the shared httpClient. -func (h *FileBrowserHandlers) newClientWithTimeout(timeout time.Duration) http.Client { - return http.Client{ - Transport: h.httpClient.Client.Transport, - Timeout: timeout, - } -} - // ShowFileBrowser renders the file browser page func (h *FileBrowserHandlers) ShowFileBrowser(w http.ResponseWriter, r *http.Request) { // Get path from query parameter, default to root @@ -309,7 +293,7 @@ func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request) } // Upload file to filer - err = h.uploadFileToFiler(fullPath, fileHeader) + err = h.uploadFileToFiler(r.Context(), fullPath, fileHeader) if err != nil { failedUploads = append(failedUploads, fmt.Sprintf("%s: %v", fileName, err)) @@ -346,139 +330,21 @@ func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request) } } -// uploadFileToFiler uploads a file directly to the filer using multipart form data -func (h *FileBrowserHandlers) uploadFileToFiler(filePath string, fileHeader *multipart.FileHeader) error { - // Get filer address from admin server - filerAddress := h.adminServer.GetFilerAddress() - if filerAddress == "" { - return fmt.Errorf("filer address not configured") - } - - // Validate and sanitize the filer address - if err := h.validateFilerAddress(filerAddress); err != nil { - return fmt.Errorf("invalid filer address: %w", err) - } - filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress() - - // Validate and sanitize the file path - cleanFilePath, err := h.validateAndCleanFilePath(filePath) - if err != nil { - return fmt.Errorf("invalid file path: %w", err) - } - - // Open the file +// uploadFileToFiler uploads a file to the cluster via filer gRPC + volume +// HTTP. This works whether or not the filer is running with -disableHttp=true, +// since the bytes never traverse the filer's HTTP listener. The multipart +// file is streamed through the chunked uploader, so the admin process never +// buffers the entire payload in memory. The caller passes the request +// context so a client disconnect cancels the in-flight chunk uploads instead +// of letting them run to completion against the volume servers. +func (h *FileBrowserHandlers) uploadFileToFiler(ctx context.Context, filePath string, fileHeader *multipart.FileHeader) error { file, err := fileHeader.Open() if err != nil { return fmt.Errorf("failed to open file: %w", err) } defer file.Close() - // Create multipart form data - var body bytes.Buffer - writer := multipart.NewWriter(&body) - - // Create form file field with normalized base filename - // Use path.Base (not filepath.Base) since cleanFilePath uses URL path semantics - baseFileName := path.Base(cleanFilePath) - part, err := writer.CreateFormFile("file", baseFileName) - if err != nil { - return fmt.Errorf("failed to create form file: %w", err) - } - - // Copy file content to form - _, err = io.Copy(part, file) - if err != nil { - return fmt.Errorf("failed to copy file content: %w", err) - } - - // Close the writer to finalize the form - err = writer.Close() - if err != nil { - return fmt.Errorf("failed to close multipart writer: %w", err) - } - - // Create the upload URL - the httpClient will normalize to the correct scheme (http/https) - // based on the https.client configuration in security.toml - uploadURL := filerFileURL(filerHttpAddress, cleanFilePath) - - // Normalize the URL scheme based on TLS configuration - uploadURL, err = h.httpClient.NormalizeHttpScheme(uploadURL) - if err != nil { - return fmt.Errorf("failed to normalize URL scheme: %w", err) - } - - // Create HTTP request - req, err := http.NewRequest("POST", uploadURL, &body) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - // Set content type with boundary - req.Header.Set("Content-Type", writer.FormDataContentType()) - - // Add JWT Token to Authorization Header - h.setupFilerJwtAuth(req, "jwt.filer_signing.key", "jwt.filer_signing.expires_after_seconds", "filer upload") - - // Send request using TLS-aware HTTP client with 60s timeout for large file uploads - // lgtm[go/ssrf] - // Safe: filerAddress validated by validateFilerAddress() to match configured filer - // Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal - client := h.newClientWithTimeout(60 * time.Second) - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("failed to upload file: %w", err) - } - defer resp.Body.Close() - - // Check response - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - responseBody, _ := io.ReadAll(resp.Body) - return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(responseBody)) - } - - return nil -} - -// validateFilerAddress validates that the filer address is safe to use -func (h *FileBrowserHandlers) validateFilerAddress(address string) error { - if address == "" { - return fmt.Errorf("filer address cannot be empty") - } - - // CRITICAL: Only allow the configured filer address to prevent SSRF - configuredFiler := h.adminServer.GetFilerAddress() - normalizedAddress := pb.ServerAddress(address).ToHttpAddress() - normalizedConfigured := pb.ServerAddress(configuredFiler).ToHttpAddress() - if normalizedAddress != normalizedConfigured { - return fmt.Errorf("address does not match configured filer: got %s, expected %s", address, configuredFiler) - } - - // Parse the normalized HTTP address to validate it's a proper host:port format. - host, port, err := net.SplitHostPort(normalizedAddress) - if err != nil { - return fmt.Errorf("invalid address format: %w", err) - } - - // Validate host is not empty - if host == "" { - return fmt.Errorf("host cannot be empty") - } - - // Validate port is numeric and in valid range - if port == "" { - return fmt.Errorf("port cannot be empty") - } - - portNum, err := strconv.Atoi(port) - if err != nil { - return fmt.Errorf("invalid port number: %w", err) - } - - if portNum < 1 || portNum > 65535 { - return fmt.Errorf("port number must be between 1 and 65535") - } - - return nil + return h.uploadFileGrpc(ctx, filePath, fileHeader.Filename, fileHeader.Header.Get("Content-Type"), file) } // validateAndCleanFilePath validates and cleans the file path to prevent path traversal @@ -507,161 +373,56 @@ func (h *FileBrowserHandlers) validateAndCleanFilePath(filePath string) (string, return cleanPath, nil } -// filerFileURL joins the filer HTTP address with a validated file path, URL-escaping -// the path so that control characters and other bytes that are legal in S3 object keys -// cannot inject into the HTTP request target. -func filerFileURL(filerHttpAddress, cleanFilePath string) string { - return filerHttpAddress + (&url.URL{Path: cleanFilePath}).EscapedPath() -} - -// fetchFileContent fetches file content from the filer and returns the content or an error. +// fetchFileContent fetches file content via the filer gRPC service. It is +// used for the "view as text" path, so the maxBytes cap matches the 1 MB +// limit the caller already applies before invoking us. func (h *FileBrowserHandlers) fetchFileContent(filePath string, timeout time.Duration) (string, error) { - filerAddress := h.adminServer.GetFilerAddress() - if filerAddress == "" { - return "", fmt.Errorf("filer address not configured") - } - - if err := h.validateFilerAddress(filerAddress); err != nil { - return "", fmt.Errorf("invalid filer address configuration: %w", err) - } - filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress() - - cleanFilePath, err := h.validateAndCleanFilePath(filePath) - if err != nil { - return "", err - } - - // Create the file URL with proper scheme based on TLS configuration - fileURL := filerFileURL(filerHttpAddress, cleanFilePath) - fileURL, err = h.httpClient.NormalizeHttpScheme(fileURL) - if err != nil { - return "", fmt.Errorf("failed to construct file URL: %w", err) - } - - // lgtm[go/ssrf] - // Safe: filerAddress validated by validateFilerAddress() to match configured filer - // Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal - client := h.newClientWithTimeout(timeout) - req, err := http.NewRequest("GET", fileURL, nil) - if err != nil { - return "", fmt.Errorf("failed to create request: %w", err) - } - h.addFilerJwtAuthHeader(req) - resp, err := client.Do(req) - if err != nil { - return "", fmt.Errorf("failed to fetch file from filer: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("filer returned status %d but failed to read response body: %w", resp.StatusCode, err) - } - return "", fmt.Errorf("filer returned status %d: %s", resp.StatusCode, string(body)) - } - - contentBytes, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("failed to read file content: %w", err) - } - - return string(contentBytes), nil + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return h.fetchFileContentGrpc(ctx, filePath, 0) } -// DownloadFile handles file download requests by proxying through the Admin UI server -// This ensures mTLS works correctly since the Admin UI server has the client certificates +// DownloadFile streams a file straight from the volume servers via the filer +// gRPC service, so the admin file browser keeps working even when the filer +// is started with -disableHttp=true. func (h *FileBrowserHandlers) DownloadFile(w http.ResponseWriter, r *http.Request) { filePath := r.URL.Query().Get("path") if filePath == "" { writeJSONError(w, http.StatusBadRequest, "File path is required") return } - - // Get filer address - filerAddress := h.adminServer.GetFilerAddress() - if filerAddress == "" { - writeJSONError(w, http.StatusInternalServerError, "Filer address not configured") - return - } - - // Validate filer address to prevent SSRF - if err := h.validateFilerAddress(filerAddress); err != nil { - writeJSONError(w, http.StatusInternalServerError, "Invalid filer address configuration") - return - } - filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress() - - // Validate and sanitize the file path - cleanFilePath, err := h.validateAndCleanFilePath(filePath) - if err != nil { - writeJSONError(w, http.StatusBadRequest, "Invalid file path: "+err.Error()) - return - } - - // Create the download URL with proper scheme based on TLS configuration - downloadURL := filerFileURL(filerHttpAddress, cleanFilePath) - downloadURL, err = h.httpClient.NormalizeHttpScheme(downloadURL) - if err != nil { - writeJSONError(w, http.StatusInternalServerError, "Failed to construct download URL: "+err.Error()) - return - } - - // Proxy the download through the Admin UI server to support mTLS - // lgtm[go/ssrf] - // Safe: filerAddress validated by validateFilerAddress() to match configured filer - // Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal - // Use request context so download is cancelled when client disconnects - req, err := http.NewRequestWithContext(r.Context(), "GET", downloadURL, nil) - if err != nil { - writeJSONError(w, http.StatusInternalServerError, "Failed to create request: "+err.Error()) - return - } - client := h.newClientWithTimeout(5 * time.Minute) // Longer timeout for large file downloads - - h.addFilerJwtAuthHeader(req) - - resp, err := client.Do(req) - if err != nil { - writeJSONError(w, http.StatusBadGateway, "Failed to fetch file from filer: "+err.Error()) - return - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, err := io.ReadAll(resp.Body) - if err != nil { - writeJSONError(w, resp.StatusCode, fmt.Sprintf("Filer returned status %d but failed to read response body: %v", resp.StatusCode, err)) + tracker := &responseWriteTracker{ResponseWriter: w} + if err := h.downloadFileGrpc(r.Context(), filePath, tracker); err != nil { + // Once bytes have been written we can't switch to a JSON error body + // without corrupting the partial response — log and stop. Before any + // write the response is still uncommitted, so a 502 with details is + // safe. + if tracker.committed { + glog.Errorf("Error streaming file download: %v", err) return } - writeJSONError(w, resp.StatusCode, fmt.Sprintf("Filer returned status %d: %s", resp.StatusCode, string(body))) - return + writeJSONError(w, http.StatusBadGateway, "Failed to fetch file: "+err.Error()) } +} - // Set headers for file download - fileName := filepath.Base(cleanFilePath) - // Use mime.FormatMediaType for RFC 6266 compliant Content-Disposition, - // properly handling non-ASCII characters and special characters - w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": fileName})) +// responseWriteTracker wraps http.ResponseWriter to record whether the +// response has been committed (status line + headers sent). DownloadFile +// uses this instead of probing Header() so future header-setting code +// reorganization can't silently break the "did we already send bytes?" +// detection. +type responseWriteTracker struct { + http.ResponseWriter + committed bool +} - // Use content type from filer response, or default to octet-stream - contentType := resp.Header.Get("Content-Type") - if contentType == "" { - contentType = "application/octet-stream" - } - w.Header().Set("Content-Type", contentType) +func (t *responseWriteTracker) WriteHeader(code int) { + t.committed = true + t.ResponseWriter.WriteHeader(code) +} - // Set content length if available - if resp.ContentLength > 0 { - w.Header().Set("Content-Length", fmt.Sprintf("%d", resp.ContentLength)) - } - - // Stream the response body to the client - w.WriteHeader(http.StatusOK) - _, err = io.Copy(w, resp.Body) - if err != nil { - glog.Errorf("Error streaming file download: %v", err) - } +func (t *responseWriteTracker) Write(p []byte) (int, error) { + t.committed = true + return t.ResponseWriter.Write(p) } // ViewFile handles file viewing requests (for text files, images, etc.) @@ -883,64 +644,16 @@ func (h *FileBrowserHandlers) formatBytes(bytes int64) string { // Helper function to check if a file is likely a text file by checking content func (h *FileBrowserHandlers) isLikelyTextFile(filePath string, maxCheckSize int64) bool { - filerAddress := h.adminServer.GetFilerAddress() - if filerAddress == "" { - return false - } - - // Validate filer address to prevent SSRF - if err := h.validateFilerAddress(filerAddress); err != nil { - glog.Errorf("Invalid filer address: %v", err) - return false - } - filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress() - - cleanFilePath, err := h.validateAndCleanFilePath(filePath) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + content, err := h.fetchFileContentGrpc(ctx, filePath, int(maxCheckSize)) if err != nil { return false } - - // Create the file URL with proper scheme based on TLS configuration - fileURL := filerFileURL(filerHttpAddress, cleanFilePath) - fileURL, err = h.httpClient.NormalizeHttpScheme(fileURL) - if err != nil { - glog.Errorf("Failed to normalize URL scheme: %v", err) - return false + if len(content) == 0 { + return true } - - // lgtm[go/ssrf] - // Safe: filerAddress validated by validateFilerAddress() to match configured filer - // Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal - client := h.newClientWithTimeout(10 * time.Second) - req, err := http.NewRequest("GET", fileURL, nil) - if err != nil { - glog.Errorf("Failed to create request: %v", err) - return false - } - h.addFilerJwtAuthHeader(req) - resp, err := client.Do(req) - if err != nil { - return false - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return false - } - - // Read first few bytes to check if it's text - buffer := make([]byte, min(maxCheckSize, 512)) - n, err := resp.Body.Read(buffer) - if err != nil && err != io.EOF { - return false - } - - if n == 0 { - return true // Empty file can be considered text - } - - // Check if content is printable text - return h.isPrintableText(buffer[:n]) + return h.isPrintableText([]byte(content)) } // Helper function to check if content is printable text @@ -973,35 +686,3 @@ func min(a, b int64) int64 { return b } -// setupFilerJwtAuth generates a JWT token and adds it to the request Authorization header if configured. -func (h *FileBrowserHandlers) setupFilerJwtAuth(req *http.Request, keyPath, expiresPath, operation string) { - // Load security configuration - v := util.GetViper() - - // Read Filer JWT token from security.toml - signingKey := security.SigningKey(v.GetString(keyPath)) - expiresAfterSec := v.GetInt(expiresPath) - - // Generate JWT token to authenticate with Filer - var jwtToken security.EncodedJwt - if len(signingKey) > 0 { - jwtToken = security.GenJwtForFilerServer(signingKey, expiresAfterSec) - glog.V(4).Infof("Generated JWT token for %s (expires in %d sec)", operation, expiresAfterSec) - } else { - if v.GetString("jwt.signing.key") != "" { - glog.Warningf("JWT %s key not configured, but general JWT security is enabled. %s without authentication.", keyPath, operation) - } else { - glog.V(1).Infof("No JWT signing key configured, %s without authentication", operation) - } - } - - // Add JWT Token to Authorization Header - if jwtToken != "" { - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", string(jwtToken))) - glog.V(4).Infof("Added JWT authorization header for %s", operation) - } -} - -func (h *FileBrowserHandlers) addFilerJwtAuthHeader(req *http.Request) { - h.setupFilerJwtAuth(req, "jwt.filer_signing.read.key", "jwt.filer_signing.read.expires_after_seconds", "filer request") -} diff --git a/weed/admin/handlers/file_browser_handlers_test.go b/weed/admin/handlers/file_browser_handlers_test.go index 583bcc1428..3baf9b3747 100644 --- a/weed/admin/handlers/file_browser_handlers_test.go +++ b/weed/admin/handlers/file_browser_handlers_test.go @@ -42,21 +42,3 @@ func TestValidateAndCleanFilePath_RejectsEmpty(t *testing.T) { } } -func TestFilerFileURL_EscapesControlChars(t *testing.T) { - cases := []struct { - addr string - path string - want string - }{ - {"http://127.0.0.1:8888", "/buckets/profilebuilder/3testGB.zip\n ", "http://127.0.0.1:8888/buckets/profilebuilder/3testGB.zip%0A%20"}, - {"http://127.0.0.1:8888", "/buckets/profilebuilder/file\rname", "http://127.0.0.1:8888/buckets/profilebuilder/file%0Dname"}, - {"http://127.0.0.1:8888", "/buckets/profilebuilder/file\x00name", "http://127.0.0.1:8888/buckets/profilebuilder/file%00name"}, - // Plain path round-trips unchanged. - {"http://h:1", "/a/b.txt", "http://h:1/a/b.txt"}, - } - for _, tc := range cases { - if got := filerFileURL(tc.addr, tc.path); got != tc.want { - t.Errorf("filerFileURL(%q, %q) = %q, want %q", tc.addr, tc.path, got, tc.want) - } - } -}