fix(admin): switch file browser upload/download to filer gRPC + volume HTTP (#9538)

* fix(admin): switch file browser upload/download to filer gRPC + volume HTTP

The admin file browser proxied uploads and downloads through the filer's
HTTP listener, so the whole feature 404'd against filers started with
-disableHttp=true even though S3 still worked on its own port. Re-route
through the filer gRPC service: LookupDirectoryEntry + StreamContent for
reads (chunks flow straight from the volume servers), AssignVolume +
volume HTTP POST + CreateEntry for writes. Volume read tokens come from
jwt.signing.read.key when configured; the old jwt.filer_signing tokens
no longer apply since the filer HTTP surface is bypassed.

* admin file browser: propagate request context + track response writes

Pass r.Context() into uploadFileToFiler so a client disconnect cancels
the in-flight chunked upload instead of letting it run to completion
against the volume servers. For DownloadFile, replace the Content-Type
probe with a small response-writer wrapper that records whether headers
or bytes have actually been sent, so the error path can't silently
convert a pre-stream failure into a partial response if future code
moves the header-setting around.
This commit is contained in:
Chris Lu authored and GitHub committed 2026-05-18 20:33:16 -07:00
1 parent 58c3fa802c
commit 7c5296dfb1
4 files changed
+359 -388

No files matched your search

+17
View File
@@ -12,6 +12,8 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"google.golang.org/grpc"
)
// WithMasterClient executes a function with a master client connection
@@ -38,6 +40,21 @@ func (s *AdminServer) WithVolumeServerClient(address pb.ServerAddress, f func(cl
})
}
// GetMasterClient returns the admin server's wdclient.MasterClient. It is used
// by file browser download paths that stream chunks straight from the volume
// servers via filer.PrepareStreamContent so they keep working when the filer
// has -disableHttp=true.
func (s *AdminServer) GetMasterClient() *wdclient.MasterClient {
return s.masterClient
}
// GetGrpcDialOption returns the dial option used for all admin-originated
// gRPC connections (TLS or insecure). File browser uploads need this when
// they perform the assign + volume HTTP POST + create-entry flow.
func (s *AdminServer) GetGrpcDialOption() grpc.DialOption {
return s.grpcDialOption
}
// GetFilerAddress returns a filer address, discovering from masters if needed
func (s *AdminServer) GetFilerAddress() string {
// Discover filers from masters
+291
View File
@@ -0,0 +1,291 @@
package handlers
import (
"bytes"
"context"
"fmt"
"io"
"mime"
"net/http"
"path"
"strconv"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// Admin file browser upload chunk sizing — kept in sync with the values
// s3api uses so files end up split into the same fid-sized pieces the rest of
// the cluster expects.
const (
adminUploadChunkSize = 8 * 1024 * 1024
adminUploadSmallFileLimit = 256 * 1024
)
// File browser handlers backed by the filer gRPC service. They bypass the
// filer's HTTP listener so the UI keeps working when the filer is started
// with -disableHttp=true; chunk bytes still flow through the volume server
// HTTP endpoints (which run on their own ports).
// fetchFileContentGrpc reads file content via the filer gRPC service, looking
// the entry up and then streaming the chunks straight from the volume servers.
// When maxBytes > 0 the stream is truncated to that many bytes — used by the
// "is this text?" sniff so unknown-MIME files don't get fully downloaded.
func (h *FileBrowserHandlers) fetchFileContentGrpc(ctx context.Context, filePath string, maxBytes int) (string, error) {
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
if err != nil {
return "", err
}
entry, err := h.lookupEntry(ctx, cleanFilePath)
if err != nil {
return "", err
}
if entry.IsDirectory {
return "", fmt.Errorf("path is a directory")
}
size := int64(filer.FileSize(entry))
streamSize := size
if maxBytes > 0 && streamSize > int64(maxBytes) {
streamSize = int64(maxBytes)
}
var buf bytes.Buffer
if err := h.streamEntryContent(ctx, entry, streamSize, &buf); err != nil {
return "", err
}
return buf.String(), nil
}
// downloadFileGrpc streams a file via gRPC + volume server HTTP. The
// response writer receives the canonical attachment headers and the raw
// bytes; this replaces the HTTP-to-filer proxy that used to run in
// DownloadFile.
func (h *FileBrowserHandlers) downloadFileGrpc(ctx context.Context, filePath string, w http.ResponseWriter) error {
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
if err != nil {
return err
}
entry, err := h.lookupEntry(ctx, cleanFilePath)
if err != nil {
return err
}
if entry.IsDirectory {
return fmt.Errorf("path is a directory")
}
size := int64(filer.FileSize(entry))
fileName := path.Base(cleanFilePath)
w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": fileName}))
contentType := ""
if entry.Attributes != nil {
contentType = entry.Attributes.Mime
}
if contentType == "" {
contentType = "application/octet-stream"
}
w.Header().Set("Content-Type", contentType)
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
w.WriteHeader(http.StatusOK)
return h.streamEntryContent(ctx, entry, size, w)
}
// uploadFileGrpc streams the upload to volume servers in 8 MiB chunks via the
// shared chunked-upload helper, then registers the assembled entry through the
// filer gRPC service. Bytes never enter the admin process's heap as a whole —
// each chunk is sized to adminUploadChunkSize. Small files (< 256 KiB) are
// stored inline on the entry, matching the S3 server's behaviour.
func (h *FileBrowserHandlers) uploadFileGrpc(ctx context.Context, filePath string, fileName string, mimeType string, reader io.Reader) error {
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
if err != nil {
return err
}
dir := path.Dir(cleanFilePath)
if dir == "." {
dir = "/"
}
entryName := path.Base(cleanFilePath)
if mimeType == "" {
mimeType = "application/octet-stream"
}
assignFunc := func(ctx context.Context, count int, expectedDataSize uint64) (*operation.VolumeAssignRequest, *operation.AssignResult, error) {
var assignResp *filer_pb.AssignVolumeResponse
err := h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
resp, assignErr := client.AssignVolume(ctx, &filer_pb.AssignVolumeRequest{
Count: int32(count),
Path: cleanFilePath,
ExpectedDataSize: expectedDataSize,
})
if assignErr != nil {
return assignErr
}
if resp.Error != "" {
return fmt.Errorf("%s", resp.Error)
}
assignResp = resp
return nil
})
if err != nil {
return nil, nil, err
}
if assignResp.Location == nil || assignResp.FileId == "" {
return nil, nil, fmt.Errorf("assign volume returned empty location")
}
return nil, &operation.AssignResult{
Fid: assignResp.FileId,
Url: assignResp.Location.Url,
PublicUrl: assignResp.Location.PublicUrl,
Count: uint64(count),
Auth: security.EncodedJwt(assignResp.Auth),
}, nil
}
chunkResult, err := operation.UploadReaderInChunks(ctx, reader, &operation.ChunkedUploadOption{
ChunkSize: adminUploadChunkSize,
SmallFileLimit: adminUploadSmallFileLimit,
SaveSmallInline: true,
MimeType: mimeType,
AssignFunc: assignFunc,
})
if err != nil {
// Partial chunks come back even on error so we can clean them up rather
// than leaving orphaned data on volume servers.
if chunkResult != nil && len(chunkResult.FileChunks) > 0 {
h.deleteOrphanedChunks(chunkResult.FileChunks)
}
return fmt.Errorf("upload: %w", err)
}
now := time.Now()
entry := &filer_pb.Entry{
Name: entryName,
Attributes: &filer_pb.FuseAttributes{
FileSize: uint64(chunkResult.TotalSize),
Mtime: now.Unix(),
Crtime: now.Unix(),
FileMode: 0644,
Mime: mimeType,
},
}
if len(chunkResult.SmallContent) > 0 {
entry.Content = chunkResult.SmallContent
} else {
entry.Chunks = chunkResult.FileChunks
}
err = h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
_, createErr := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{
Directory: dir,
Entry: entry,
})
return createErr
})
if err != nil {
if len(chunkResult.FileChunks) > 0 {
h.deleteOrphanedChunks(chunkResult.FileChunks)
}
return fmt.Errorf("create entry: %w", err)
}
return nil
}
// deleteOrphanedChunks best-effort removes the chunk fids when an upload
// fails partway through. Errors are logged; we can't surface them past the
// caller's primary failure.
func (h *FileBrowserHandlers) deleteOrphanedChunks(chunks []*filer_pb.FileChunk) {
fileIds := make([]string, 0, len(chunks))
for _, c := range chunks {
if fid := c.GetFileIdString(); fid != "" {
fileIds = append(fileIds, fid)
}
}
if len(fileIds) == 0 {
return
}
master := h.adminServer.GetMasterClient()
results := operation.DeleteFileIds(master.GetMaster, false, h.adminServer.GetGrpcDialOption(), fileIds)
for _, r := range results {
if r.Error != "" {
glog.Warningf("admin file browser: orphan chunk %s cleanup: %s", r.FileId, r.Error)
}
}
}
func (h *FileBrowserHandlers) lookupEntry(ctx context.Context, cleanFilePath string) (*filer_pb.Entry, error) {
dir := path.Dir(cleanFilePath)
if dir == "." {
dir = "/"
}
name := path.Base(cleanFilePath)
var entry *filer_pb.Entry
err := h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
resp, lookupErr := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if lookupErr != nil {
return lookupErr
}
if resp.Entry == nil {
return fmt.Errorf("not found")
}
entry = resp.Entry
return nil
})
return entry, err
}
func (h *FileBrowserHandlers) streamEntryContent(ctx context.Context, entry *filer_pb.Entry, size int64, w io.Writer) error {
if size == 0 {
// Inline content (small files stored directly on the entry) skip the
// chunk pipeline entirely.
if len(entry.Content) > 0 {
_, err := w.Write(entry.Content)
return err
}
return nil
}
if len(entry.Content) > 0 && len(entry.GetChunks()) == 0 {
_, err := w.Write(entry.Content)
return err
}
streamFn, err := filer.PrepareStreamContentWithThrottler(
ctx,
h.adminServer.GetMasterClient(),
volumeServerReadJwt,
entry.GetChunks(),
0,
size,
0,
)
if err != nil {
return fmt.Errorf("prepare stream: %w", err)
}
return streamFn(w)
}
// volumeServerReadJwt mints a per-fileId Bearer token for reads against a
// volume server when jwt.signing.read.key is configured. The volume servers
// are unaware of jwt.filer_signing.read.key — that one only gates the filer
// HTTP surface, which this code path doesn't touch.
func volumeServerReadJwt(fileId string) string {
v := util.GetViper()
signingKey := security.SigningKey(v.GetString("jwt.signing.read.key"))
if len(signingKey) == 0 {
return ""
}
expiresAfterSec := v.GetInt("jwt.signing.read.expires_after_seconds")
return string(security.GenJwtForVolumeServer(signingKey, expiresAfterSec, fileId))
}
+51 -370
View File
@@ -1,15 +1,10 @@
package handlers
import (
"bytes"
"context"
"fmt"
"io"
"mime"
"mime/multipart"
"net"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
@@ -21,9 +16,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/http/client"
)
@@ -49,15 +42,6 @@ func NewFileBrowserHandlers(adminServer *dash.AdminServer) *FileBrowserHandlers
}
}
// newClientWithTimeout creates a temporary http.Client with the specified timeout,
// reusing the TLS transport from the shared httpClient.
func (h *FileBrowserHandlers) newClientWithTimeout(timeout time.Duration) http.Client {
return http.Client{
Transport: h.httpClient.Client.Transport,
Timeout: timeout,
}
}
// ShowFileBrowser renders the file browser page
func (h *FileBrowserHandlers) ShowFileBrowser(w http.ResponseWriter, r *http.Request) {
// Get path from query parameter, default to root
@@ -309,7 +293,7 @@ func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request)
}
// Upload file to filer
err = h.uploadFileToFiler(fullPath, fileHeader)
err = h.uploadFileToFiler(r.Context(), fullPath, fileHeader)
if err != nil {
failedUploads = append(failedUploads, fmt.Sprintf("%s: %v", fileName, err))
@@ -346,139 +330,21 @@ func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request)
}
}
// uploadFileToFiler uploads a file directly to the filer using multipart form data
func (h *FileBrowserHandlers) uploadFileToFiler(filePath string, fileHeader *multipart.FileHeader) error {
// Get filer address from admin server
filerAddress := h.adminServer.GetFilerAddress()
if filerAddress == "" {
return fmt.Errorf("filer address not configured")
}
// Validate and sanitize the filer address
if err := h.validateFilerAddress(filerAddress); err != nil {
return fmt.Errorf("invalid filer address: %w", err)
}
filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress()
// Validate and sanitize the file path
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
if err != nil {
return fmt.Errorf("invalid file path: %w", err)
}
// Open the file
// uploadFileToFiler uploads a file to the cluster via filer gRPC + volume
// HTTP. This works whether or not the filer is running with -disableHttp=true,
// since the bytes never traverse the filer's HTTP listener. The multipart
// file is streamed through the chunked uploader, so the admin process never
// buffers the entire payload in memory. The caller passes the request
// context so a client disconnect cancels the in-flight chunk uploads instead
// of letting them run to completion against the volume servers.
func (h *FileBrowserHandlers) uploadFileToFiler(ctx context.Context, filePath string, fileHeader *multipart.FileHeader) error {
file, err := fileHeader.Open()
if err != nil {
return fmt.Errorf("failed to open file: %w", err)
}
defer file.Close()
// Create multipart form data
var body bytes.Buffer
writer := multipart.NewWriter(&body)
// Create form file field with normalized base filename
// Use path.Base (not filepath.Base) since cleanFilePath uses URL path semantics
baseFileName := path.Base(cleanFilePath)
part, err := writer.CreateFormFile("file", baseFileName)
if err != nil {
return fmt.Errorf("failed to create form file: %w", err)
}
// Copy file content to form
_, err = io.Copy(part, file)
if err != nil {
return fmt.Errorf("failed to copy file content: %w", err)
}
// Close the writer to finalize the form
err = writer.Close()
if err != nil {
return fmt.Errorf("failed to close multipart writer: %w", err)
}
// Create the upload URL - the httpClient will normalize to the correct scheme (http/https)
// based on the https.client configuration in security.toml
uploadURL := filerFileURL(filerHttpAddress, cleanFilePath)
// Normalize the URL scheme based on TLS configuration
uploadURL, err = h.httpClient.NormalizeHttpScheme(uploadURL)
if err != nil {
return fmt.Errorf("failed to normalize URL scheme: %w", err)
}
// Create HTTP request
req, err := http.NewRequest("POST", uploadURL, &body)
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
// Set content type with boundary
req.Header.Set("Content-Type", writer.FormDataContentType())
// Add JWT Token to Authorization Header
h.setupFilerJwtAuth(req, "jwt.filer_signing.key", "jwt.filer_signing.expires_after_seconds", "filer upload")
// Send request using TLS-aware HTTP client with 60s timeout for large file uploads
// lgtm[go/ssrf]
// Safe: filerAddress validated by validateFilerAddress() to match configured filer
// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
client := h.newClientWithTimeout(60 * time.Second)
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to upload file: %w", err)
}
defer resp.Body.Close()
// Check response
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
responseBody, _ := io.ReadAll(resp.Body)
return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(responseBody))
}
return nil
}
// validateFilerAddress validates that the filer address is safe to use
func (h *FileBrowserHandlers) validateFilerAddress(address string) error {
if address == "" {
return fmt.Errorf("filer address cannot be empty")
}
// CRITICAL: Only allow the configured filer address to prevent SSRF
configuredFiler := h.adminServer.GetFilerAddress()
normalizedAddress := pb.ServerAddress(address).ToHttpAddress()
normalizedConfigured := pb.ServerAddress(configuredFiler).ToHttpAddress()
if normalizedAddress != normalizedConfigured {
return fmt.Errorf("address does not match configured filer: got %s, expected %s", address, configuredFiler)
}
// Parse the normalized HTTP address to validate it's a proper host:port format.
host, port, err := net.SplitHostPort(normalizedAddress)
if err != nil {
return fmt.Errorf("invalid address format: %w", err)
}
// Validate host is not empty
if host == "" {
return fmt.Errorf("host cannot be empty")
}
// Validate port is numeric and in valid range
if port == "" {
return fmt.Errorf("port cannot be empty")
}
portNum, err := strconv.Atoi(port)
if err != nil {
return fmt.Errorf("invalid port number: %w", err)
}
if portNum < 1 || portNum > 65535 {
return fmt.Errorf("port number must be between 1 and 65535")
}
return nil
return h.uploadFileGrpc(ctx, filePath, fileHeader.Filename, fileHeader.Header.Get("Content-Type"), file)
}
// validateAndCleanFilePath validates and cleans the file path to prevent path traversal
@@ -507,161 +373,56 @@ func (h *FileBrowserHandlers) validateAndCleanFilePath(filePath string) (string,
return cleanPath, nil
}
// filerFileURL joins the filer HTTP address with a validated file path, URL-escaping
// the path so that control characters and other bytes that are legal in S3 object keys
// cannot inject into the HTTP request target.
func filerFileURL(filerHttpAddress, cleanFilePath string) string {
return filerHttpAddress + (&url.URL{Path: cleanFilePath}).EscapedPath()
}
// fetchFileContent fetches file content from the filer and returns the content or an error.
// fetchFileContent fetches file content via the filer gRPC service. It is
// used for the "view as text" path, so the maxBytes cap matches the 1 MB
// limit the caller already applies before invoking us.
func (h *FileBrowserHandlers) fetchFileContent(filePath string, timeout time.Duration) (string, error) {
filerAddress := h.adminServer.GetFilerAddress()
if filerAddress == "" {
return "", fmt.Errorf("filer address not configured")
}
if err := h.validateFilerAddress(filerAddress); err != nil {
return "", fmt.Errorf("invalid filer address configuration: %w", err)
}
filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress()
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
if err != nil {
return "", err
}
// Create the file URL with proper scheme based on TLS configuration
fileURL := filerFileURL(filerHttpAddress, cleanFilePath)
fileURL, err = h.httpClient.NormalizeHttpScheme(fileURL)
if err != nil {
return "", fmt.Errorf("failed to construct file URL: %w", err)
}
// lgtm[go/ssrf]
// Safe: filerAddress validated by validateFilerAddress() to match configured filer
// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
client := h.newClientWithTimeout(timeout)
req, err := http.NewRequest("GET", fileURL, nil)
if err != nil {
return "", fmt.Errorf("failed to create request: %w", err)
}
h.addFilerJwtAuthHeader(req)
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("failed to fetch file from filer: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("filer returned status %d but failed to read response body: %w", resp.StatusCode, err)
}
return "", fmt.Errorf("filer returned status %d: %s", resp.StatusCode, string(body))
}
contentBytes, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("failed to read file content: %w", err)
}
return string(contentBytes), nil
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return h.fetchFileContentGrpc(ctx, filePath, 0)
}
// DownloadFile handles file download requests by proxying through the Admin UI server
// This ensures mTLS works correctly since the Admin UI server has the client certificates
// DownloadFile streams a file straight from the volume servers via the filer
// gRPC service, so the admin file browser keeps working even when the filer
// is started with -disableHttp=true.
func (h *FileBrowserHandlers) DownloadFile(w http.ResponseWriter, r *http.Request) {
filePath := r.URL.Query().Get("path")
if filePath == "" {
writeJSONError(w, http.StatusBadRequest, "File path is required")
return
}
// Get filer address
filerAddress := h.adminServer.GetFilerAddress()
if filerAddress == "" {
writeJSONError(w, http.StatusInternalServerError, "Filer address not configured")
return
}
// Validate filer address to prevent SSRF
if err := h.validateFilerAddress(filerAddress); err != nil {
writeJSONError(w, http.StatusInternalServerError, "Invalid filer address configuration")
return
}
filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress()
// Validate and sanitize the file path
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
if err != nil {
writeJSONError(w, http.StatusBadRequest, "Invalid file path: "+err.Error())
return
}
// Create the download URL with proper scheme based on TLS configuration
downloadURL := filerFileURL(filerHttpAddress, cleanFilePath)
downloadURL, err = h.httpClient.NormalizeHttpScheme(downloadURL)
if err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to construct download URL: "+err.Error())
return
}
// Proxy the download through the Admin UI server to support mTLS
// lgtm[go/ssrf]
// Safe: filerAddress validated by validateFilerAddress() to match configured filer
// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
// Use request context so download is cancelled when client disconnects
req, err := http.NewRequestWithContext(r.Context(), "GET", downloadURL, nil)
if err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to create request: "+err.Error())
return
}
client := h.newClientWithTimeout(5 * time.Minute) // Longer timeout for large file downloads
h.addFilerJwtAuthHeader(req)
resp, err := client.Do(req)
if err != nil {
writeJSONError(w, http.StatusBadGateway, "Failed to fetch file from filer: "+err.Error())
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, err := io.ReadAll(resp.Body)
if err != nil {
writeJSONError(w, resp.StatusCode, fmt.Sprintf("Filer returned status %d but failed to read response body: %v", resp.StatusCode, err))
tracker := &responseWriteTracker{ResponseWriter: w}
if err := h.downloadFileGrpc(r.Context(), filePath, tracker); err != nil {
// Once bytes have been written we can't switch to a JSON error body
// without corrupting the partial response — log and stop. Before any
// write the response is still uncommitted, so a 502 with details is
// safe.
if tracker.committed {
glog.Errorf("Error streaming file download: %v", err)
return
}
writeJSONError(w, resp.StatusCode, fmt.Sprintf("Filer returned status %d: %s", resp.StatusCode, string(body)))
return
writeJSONError(w, http.StatusBadGateway, "Failed to fetch file: "+err.Error())
}
}
// Set headers for file download
fileName := filepath.Base(cleanFilePath)
// Use mime.FormatMediaType for RFC 6266 compliant Content-Disposition,
// properly handling non-ASCII characters and special characters
w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": fileName}))
// responseWriteTracker wraps http.ResponseWriter to record whether the
// response has been committed (status line + headers sent). DownloadFile
// uses this instead of probing Header() so future header-setting code
// reorganization can't silently break the "did we already send bytes?"
// detection.
type responseWriteTracker struct {
http.ResponseWriter
committed bool
}
// Use content type from filer response, or default to octet-stream
contentType := resp.Header.Get("Content-Type")
if contentType == "" {
contentType = "application/octet-stream"
}
w.Header().Set("Content-Type", contentType)
func (t *responseWriteTracker) WriteHeader(code int) {
t.committed = true
t.ResponseWriter.WriteHeader(code)
}
// Set content length if available
if resp.ContentLength > 0 {
w.Header().Set("Content-Length", fmt.Sprintf("%d", resp.ContentLength))
}
// Stream the response body to the client
w.WriteHeader(http.StatusOK)
_, err = io.Copy(w, resp.Body)
if err != nil {
glog.Errorf("Error streaming file download: %v", err)
}
func (t *responseWriteTracker) Write(p []byte) (int, error) {
t.committed = true
return t.ResponseWriter.Write(p)
}
// ViewFile handles file viewing requests (for text files, images, etc.)
@@ -883,64 +644,16 @@ func (h *FileBrowserHandlers) formatBytes(bytes int64) string {
// Helper function to check if a file is likely a text file by checking content
func (h *FileBrowserHandlers) isLikelyTextFile(filePath string, maxCheckSize int64) bool {
filerAddress := h.adminServer.GetFilerAddress()
if filerAddress == "" {
return false
}
// Validate filer address to prevent SSRF
if err := h.validateFilerAddress(filerAddress); err != nil {
glog.Errorf("Invalid filer address: %v", err)
return false
}
filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress()
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
content, err := h.fetchFileContentGrpc(ctx, filePath, int(maxCheckSize))
if err != nil {
return false
}
// Create the file URL with proper scheme based on TLS configuration
fileURL := filerFileURL(filerHttpAddress, cleanFilePath)
fileURL, err = h.httpClient.NormalizeHttpScheme(fileURL)
if err != nil {
glog.Errorf("Failed to normalize URL scheme: %v", err)
return false
if len(content) == 0 {
return true
}
// lgtm[go/ssrf]
// Safe: filerAddress validated by validateFilerAddress() to match configured filer
// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
client := h.newClientWithTimeout(10 * time.Second)
req, err := http.NewRequest("GET", fileURL, nil)
if err != nil {
glog.Errorf("Failed to create request: %v", err)
return false
}
h.addFilerJwtAuthHeader(req)
resp, err := client.Do(req)
if err != nil {
return false
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return false
}
// Read first few bytes to check if it's text
buffer := make([]byte, min(maxCheckSize, 512))
n, err := resp.Body.Read(buffer)
if err != nil && err != io.EOF {
return false
}
if n == 0 {
return true // Empty file can be considered text
}
// Check if content is printable text
return h.isPrintableText(buffer[:n])
return h.isPrintableText([]byte(content))
}
// Helper function to check if content is printable text
@@ -973,35 +686,3 @@ func min(a, b int64) int64 {
return b
}
// setupFilerJwtAuth generates a JWT token and adds it to the request Authorization header if configured.
func (h *FileBrowserHandlers) setupFilerJwtAuth(req *http.Request, keyPath, expiresPath, operation string) {
// Load security configuration
v := util.GetViper()
// Read Filer JWT token from security.toml
signingKey := security.SigningKey(v.GetString(keyPath))
expiresAfterSec := v.GetInt(expiresPath)
// Generate JWT token to authenticate with Filer
var jwtToken security.EncodedJwt
if len(signingKey) > 0 {
jwtToken = security.GenJwtForFilerServer(signingKey, expiresAfterSec)
glog.V(4).Infof("Generated JWT token for %s (expires in %d sec)", operation, expiresAfterSec)
} else {
if v.GetString("jwt.signing.key") != "" {
glog.Warningf("JWT %s key not configured, but general JWT security is enabled. %s without authentication.", keyPath, operation)
} else {
glog.V(1).Infof("No JWT signing key configured, %s without authentication", operation)
}
}
// Add JWT Token to Authorization Header
if jwtToken != "" {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", string(jwtToken)))
glog.V(4).Infof("Added JWT authorization header for %s", operation)
}
}
func (h *FileBrowserHandlers) addFilerJwtAuthHeader(req *http.Request) {
h.setupFilerJwtAuth(req, "jwt.filer_signing.read.key", "jwt.filer_signing.read.expires_after_seconds", "filer request")
}
@@ -42,21 +42,3 @@ func TestValidateAndCleanFilePath_RejectsEmpty(t *testing.T) {
}
}
func TestFilerFileURL_EscapesControlChars(t *testing.T) {
cases := []struct {
addr string
path string
want string
}{
{"http://127.0.0.1:8888", "/buckets/profilebuilder/3testGB.zip\n ", "http://127.0.0.1:8888/buckets/profilebuilder/3testGB.zip%0A%20"},
{"http://127.0.0.1:8888", "/buckets/profilebuilder/file\rname", "http://127.0.0.1:8888/buckets/profilebuilder/file%0Dname"},
{"http://127.0.0.1:8888", "/buckets/profilebuilder/file\x00name", "http://127.0.0.1:8888/buckets/profilebuilder/file%00name"},
// Plain path round-trips unchanged.
{"http://h:1", "/a/b.txt", "http://h:1/a/b.txt"},
}
for _, tc := range cases {
if got := filerFileURL(tc.addr, tc.path); got != tc.want {
t.Errorf("filerFileURL(%q, %q) = %q, want %q", tc.addr, tc.path, got, tc.want)
}
}
}