mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-22 22:11:39 +02:00
fix(admin): switch file browser upload/download to filer gRPC + volume HTTP (#9538)
* fix(admin): switch file browser upload/download to filer gRPC + volume HTTP The admin file browser proxied uploads and downloads through the filer's HTTP listener, so the whole feature 404'd against filers started with -disableHttp=true even though S3 still worked on its own port. Re-route through the filer gRPC service: LookupDirectoryEntry + StreamContent for reads (chunks flow straight from the volume servers), AssignVolume + volume HTTP POST + CreateEntry for writes. Volume read tokens come from jwt.signing.read.key when configured; the old jwt.filer_signing tokens no longer apply since the filer HTTP surface is bypassed. * admin file browser: propagate request context + track response writes Pass r.Context() into uploadFileToFiler so a client disconnect cancels the in-flight chunked upload instead of letting it run to completion against the volume servers. For DownloadFile, replace the Content-Type probe with a small response-writer wrapper that records whether headers or bytes have actually been sent, so the error path can't silently convert a pre-stream failure into a partial response if future code moves the header-setting around.
This commit is contained in:
1 parent
58c3fa802c
commit
7c5296dfb1
4 files changed
+359
-388
No files matched your search
@@ -12,6 +12,8 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
// WithMasterClient executes a function with a master client connection
|
||||
@@ -38,6 +40,21 @@ func (s *AdminServer) WithVolumeServerClient(address pb.ServerAddress, f func(cl
|
||||
})
|
||||
}
|
||||
|
||||
// GetMasterClient returns the admin server's wdclient.MasterClient. It is used
|
||||
// by file browser download paths that stream chunks straight from the volume
|
||||
// servers via filer.PrepareStreamContent so they keep working when the filer
|
||||
// has -disableHttp=true.
|
||||
func (s *AdminServer) GetMasterClient() *wdclient.MasterClient {
|
||||
return s.masterClient
|
||||
}
|
||||
|
||||
// GetGrpcDialOption returns the dial option used for all admin-originated
|
||||
// gRPC connections (TLS or insecure). File browser uploads need this when
|
||||
// they perform the assign + volume HTTP POST + create-entry flow.
|
||||
func (s *AdminServer) GetGrpcDialOption() grpc.DialOption {
|
||||
return s.grpcDialOption
|
||||
}
|
||||
|
||||
// GetFilerAddress returns a filer address, discovering from masters if needed
|
||||
func (s *AdminServer) GetFilerAddress() string {
|
||||
// Discover filers from masters
|
||||
|
||||
@@ -0,0 +1,291 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
// Admin file browser upload chunk sizing — kept in sync with the values
|
||||
// s3api uses so files end up split into the same fid-sized pieces the rest of
|
||||
// the cluster expects.
|
||||
const (
|
||||
adminUploadChunkSize = 8 * 1024 * 1024
|
||||
adminUploadSmallFileLimit = 256 * 1024
|
||||
)
|
||||
|
||||
// File browser handlers backed by the filer gRPC service. They bypass the
|
||||
// filer's HTTP listener so the UI keeps working when the filer is started
|
||||
// with -disableHttp=true; chunk bytes still flow through the volume server
|
||||
// HTTP endpoints (which run on their own ports).
|
||||
|
||||
// fetchFileContentGrpc reads file content via the filer gRPC service, looking
|
||||
// the entry up and then streaming the chunks straight from the volume servers.
|
||||
// When maxBytes > 0 the stream is truncated to that many bytes — used by the
|
||||
// "is this text?" sniff so unknown-MIME files don't get fully downloaded.
|
||||
func (h *FileBrowserHandlers) fetchFileContentGrpc(ctx context.Context, filePath string, maxBytes int) (string, error) {
|
||||
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
entry, err := h.lookupEntry(ctx, cleanFilePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if entry.IsDirectory {
|
||||
return "", fmt.Errorf("path is a directory")
|
||||
}
|
||||
|
||||
size := int64(filer.FileSize(entry))
|
||||
streamSize := size
|
||||
if maxBytes > 0 && streamSize > int64(maxBytes) {
|
||||
streamSize = int64(maxBytes)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := h.streamEntryContent(ctx, entry, streamSize, &buf); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
// downloadFileGrpc streams a file via gRPC + volume server HTTP. The
|
||||
// response writer receives the canonical attachment headers and the raw
|
||||
// bytes; this replaces the HTTP-to-filer proxy that used to run in
|
||||
// DownloadFile.
|
||||
func (h *FileBrowserHandlers) downloadFileGrpc(ctx context.Context, filePath string, w http.ResponseWriter) error {
|
||||
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
entry, err := h.lookupEntry(ctx, cleanFilePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDirectory {
|
||||
return fmt.Errorf("path is a directory")
|
||||
}
|
||||
|
||||
size := int64(filer.FileSize(entry))
|
||||
|
||||
fileName := path.Base(cleanFilePath)
|
||||
w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": fileName}))
|
||||
|
||||
contentType := ""
|
||||
if entry.Attributes != nil {
|
||||
contentType = entry.Attributes.Mime
|
||||
}
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
return h.streamEntryContent(ctx, entry, size, w)
|
||||
}
|
||||
|
||||
// uploadFileGrpc streams the upload to volume servers in 8 MiB chunks via the
|
||||
// shared chunked-upload helper, then registers the assembled entry through the
|
||||
// filer gRPC service. Bytes never enter the admin process's heap as a whole —
|
||||
// each chunk is sized to adminUploadChunkSize. Small files (< 256 KiB) are
|
||||
// stored inline on the entry, matching the S3 server's behaviour.
|
||||
func (h *FileBrowserHandlers) uploadFileGrpc(ctx context.Context, filePath string, fileName string, mimeType string, reader io.Reader) error {
|
||||
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dir := path.Dir(cleanFilePath)
|
||||
if dir == "." {
|
||||
dir = "/"
|
||||
}
|
||||
entryName := path.Base(cleanFilePath)
|
||||
if mimeType == "" {
|
||||
mimeType = "application/octet-stream"
|
||||
}
|
||||
|
||||
assignFunc := func(ctx context.Context, count int, expectedDataSize uint64) (*operation.VolumeAssignRequest, *operation.AssignResult, error) {
|
||||
var assignResp *filer_pb.AssignVolumeResponse
|
||||
err := h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, assignErr := client.AssignVolume(ctx, &filer_pb.AssignVolumeRequest{
|
||||
Count: int32(count),
|
||||
Path: cleanFilePath,
|
||||
ExpectedDataSize: expectedDataSize,
|
||||
})
|
||||
if assignErr != nil {
|
||||
return assignErr
|
||||
}
|
||||
if resp.Error != "" {
|
||||
return fmt.Errorf("%s", resp.Error)
|
||||
}
|
||||
assignResp = resp
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if assignResp.Location == nil || assignResp.FileId == "" {
|
||||
return nil, nil, fmt.Errorf("assign volume returned empty location")
|
||||
}
|
||||
return nil, &operation.AssignResult{
|
||||
Fid: assignResp.FileId,
|
||||
Url: assignResp.Location.Url,
|
||||
PublicUrl: assignResp.Location.PublicUrl,
|
||||
Count: uint64(count),
|
||||
Auth: security.EncodedJwt(assignResp.Auth),
|
||||
}, nil
|
||||
}
|
||||
|
||||
chunkResult, err := operation.UploadReaderInChunks(ctx, reader, &operation.ChunkedUploadOption{
|
||||
ChunkSize: adminUploadChunkSize,
|
||||
SmallFileLimit: adminUploadSmallFileLimit,
|
||||
SaveSmallInline: true,
|
||||
MimeType: mimeType,
|
||||
AssignFunc: assignFunc,
|
||||
})
|
||||
if err != nil {
|
||||
// Partial chunks come back even on error so we can clean them up rather
|
||||
// than leaving orphaned data on volume servers.
|
||||
if chunkResult != nil && len(chunkResult.FileChunks) > 0 {
|
||||
h.deleteOrphanedChunks(chunkResult.FileChunks)
|
||||
}
|
||||
return fmt.Errorf("upload: %w", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
entry := &filer_pb.Entry{
|
||||
Name: entryName,
|
||||
Attributes: &filer_pb.FuseAttributes{
|
||||
FileSize: uint64(chunkResult.TotalSize),
|
||||
Mtime: now.Unix(),
|
||||
Crtime: now.Unix(),
|
||||
FileMode: 0644,
|
||||
Mime: mimeType,
|
||||
},
|
||||
}
|
||||
if len(chunkResult.SmallContent) > 0 {
|
||||
entry.Content = chunkResult.SmallContent
|
||||
} else {
|
||||
entry.Chunks = chunkResult.FileChunks
|
||||
}
|
||||
|
||||
err = h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
_, createErr := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{
|
||||
Directory: dir,
|
||||
Entry: entry,
|
||||
})
|
||||
return createErr
|
||||
})
|
||||
if err != nil {
|
||||
if len(chunkResult.FileChunks) > 0 {
|
||||
h.deleteOrphanedChunks(chunkResult.FileChunks)
|
||||
}
|
||||
return fmt.Errorf("create entry: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteOrphanedChunks best-effort removes the chunk fids when an upload
|
||||
// fails partway through. Errors are logged; we can't surface them past the
|
||||
// caller's primary failure.
|
||||
func (h *FileBrowserHandlers) deleteOrphanedChunks(chunks []*filer_pb.FileChunk) {
|
||||
fileIds := make([]string, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
if fid := c.GetFileIdString(); fid != "" {
|
||||
fileIds = append(fileIds, fid)
|
||||
}
|
||||
}
|
||||
if len(fileIds) == 0 {
|
||||
return
|
||||
}
|
||||
master := h.adminServer.GetMasterClient()
|
||||
results := operation.DeleteFileIds(master.GetMaster, false, h.adminServer.GetGrpcDialOption(), fileIds)
|
||||
for _, r := range results {
|
||||
if r.Error != "" {
|
||||
glog.Warningf("admin file browser: orphan chunk %s cleanup: %s", r.FileId, r.Error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *FileBrowserHandlers) lookupEntry(ctx context.Context, cleanFilePath string) (*filer_pb.Entry, error) {
|
||||
dir := path.Dir(cleanFilePath)
|
||||
if dir == "." {
|
||||
dir = "/"
|
||||
}
|
||||
name := path.Base(cleanFilePath)
|
||||
var entry *filer_pb.Entry
|
||||
err := h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, lookupErr := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
})
|
||||
if lookupErr != nil {
|
||||
return lookupErr
|
||||
}
|
||||
if resp.Entry == nil {
|
||||
return fmt.Errorf("not found")
|
||||
}
|
||||
entry = resp.Entry
|
||||
return nil
|
||||
})
|
||||
return entry, err
|
||||
}
|
||||
|
||||
func (h *FileBrowserHandlers) streamEntryContent(ctx context.Context, entry *filer_pb.Entry, size int64, w io.Writer) error {
|
||||
if size == 0 {
|
||||
// Inline content (small files stored directly on the entry) skip the
|
||||
// chunk pipeline entirely.
|
||||
if len(entry.Content) > 0 {
|
||||
_, err := w.Write(entry.Content)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if len(entry.Content) > 0 && len(entry.GetChunks()) == 0 {
|
||||
_, err := w.Write(entry.Content)
|
||||
return err
|
||||
}
|
||||
|
||||
streamFn, err := filer.PrepareStreamContentWithThrottler(
|
||||
ctx,
|
||||
h.adminServer.GetMasterClient(),
|
||||
volumeServerReadJwt,
|
||||
entry.GetChunks(),
|
||||
0,
|
||||
size,
|
||||
0,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare stream: %w", err)
|
||||
}
|
||||
return streamFn(w)
|
||||
}
|
||||
|
||||
// volumeServerReadJwt mints a per-fileId Bearer token for reads against a
|
||||
// volume server when jwt.signing.read.key is configured. The volume servers
|
||||
// are unaware of jwt.filer_signing.read.key — that one only gates the filer
|
||||
// HTTP surface, which this code path doesn't touch.
|
||||
func volumeServerReadJwt(fileId string) string {
|
||||
v := util.GetViper()
|
||||
signingKey := security.SigningKey(v.GetString("jwt.signing.read.key"))
|
||||
if len(signingKey) == 0 {
|
||||
return ""
|
||||
}
|
||||
expiresAfterSec := v.GetInt("jwt.signing.read.expires_after_seconds")
|
||||
return string(security.GenJwtForVolumeServer(signingKey, expiresAfterSec, fileId))
|
||||
}
|
||||
@@ -1,15 +1,10 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -21,9 +16,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/http/client"
|
||||
)
|
||||
@@ -49,15 +42,6 @@ func NewFileBrowserHandlers(adminServer *dash.AdminServer) *FileBrowserHandlers
|
||||
}
|
||||
}
|
||||
|
||||
// newClientWithTimeout creates a temporary http.Client with the specified timeout,
|
||||
// reusing the TLS transport from the shared httpClient.
|
||||
func (h *FileBrowserHandlers) newClientWithTimeout(timeout time.Duration) http.Client {
|
||||
return http.Client{
|
||||
Transport: h.httpClient.Client.Transport,
|
||||
Timeout: timeout,
|
||||
}
|
||||
}
|
||||
|
||||
// ShowFileBrowser renders the file browser page
|
||||
func (h *FileBrowserHandlers) ShowFileBrowser(w http.ResponseWriter, r *http.Request) {
|
||||
// Get path from query parameter, default to root
|
||||
@@ -309,7 +293,7 @@ func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
// Upload file to filer
|
||||
err = h.uploadFileToFiler(fullPath, fileHeader)
|
||||
err = h.uploadFileToFiler(r.Context(), fullPath, fileHeader)
|
||||
|
||||
if err != nil {
|
||||
failedUploads = append(failedUploads, fmt.Sprintf("%s: %v", fileName, err))
|
||||
@@ -346,139 +330,21 @@ func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
|
||||
// uploadFileToFiler uploads a file directly to the filer using multipart form data
|
||||
func (h *FileBrowserHandlers) uploadFileToFiler(filePath string, fileHeader *multipart.FileHeader) error {
|
||||
// Get filer address from admin server
|
||||
filerAddress := h.adminServer.GetFilerAddress()
|
||||
if filerAddress == "" {
|
||||
return fmt.Errorf("filer address not configured")
|
||||
}
|
||||
|
||||
// Validate and sanitize the filer address
|
||||
if err := h.validateFilerAddress(filerAddress); err != nil {
|
||||
return fmt.Errorf("invalid filer address: %w", err)
|
||||
}
|
||||
filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress()
|
||||
|
||||
// Validate and sanitize the file path
|
||||
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
// Open the file
|
||||
// uploadFileToFiler uploads a file to the cluster via filer gRPC + volume
|
||||
// HTTP. This works whether or not the filer is running with -disableHttp=true,
|
||||
// since the bytes never traverse the filer's HTTP listener. The multipart
|
||||
// file is streamed through the chunked uploader, so the admin process never
|
||||
// buffers the entire payload in memory. The caller passes the request
|
||||
// context so a client disconnect cancels the in-flight chunk uploads instead
|
||||
// of letting them run to completion against the volume servers.
|
||||
func (h *FileBrowserHandlers) uploadFileToFiler(ctx context.Context, filePath string, fileHeader *multipart.FileHeader) error {
|
||||
file, err := fileHeader.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Create multipart form data
|
||||
var body bytes.Buffer
|
||||
writer := multipart.NewWriter(&body)
|
||||
|
||||
// Create form file field with normalized base filename
|
||||
// Use path.Base (not filepath.Base) since cleanFilePath uses URL path semantics
|
||||
baseFileName := path.Base(cleanFilePath)
|
||||
part, err := writer.CreateFormFile("file", baseFileName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create form file: %w", err)
|
||||
}
|
||||
|
||||
// Copy file content to form
|
||||
_, err = io.Copy(part, file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy file content: %w", err)
|
||||
}
|
||||
|
||||
// Close the writer to finalize the form
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to close multipart writer: %w", err)
|
||||
}
|
||||
|
||||
// Create the upload URL - the httpClient will normalize to the correct scheme (http/https)
|
||||
// based on the https.client configuration in security.toml
|
||||
uploadURL := filerFileURL(filerHttpAddress, cleanFilePath)
|
||||
|
||||
// Normalize the URL scheme based on TLS configuration
|
||||
uploadURL, err = h.httpClient.NormalizeHttpScheme(uploadURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to normalize URL scheme: %w", err)
|
||||
}
|
||||
|
||||
// Create HTTP request
|
||||
req, err := http.NewRequest("POST", uploadURL, &body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
// Set content type with boundary
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
// Add JWT Token to Authorization Header
|
||||
h.setupFilerJwtAuth(req, "jwt.filer_signing.key", "jwt.filer_signing.expires_after_seconds", "filer upload")
|
||||
|
||||
// Send request using TLS-aware HTTP client with 60s timeout for large file uploads
|
||||
// lgtm[go/ssrf]
|
||||
// Safe: filerAddress validated by validateFilerAddress() to match configured filer
|
||||
// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
|
||||
client := h.newClientWithTimeout(60 * time.Second)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload file: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Check response
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
|
||||
responseBody, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(responseBody))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateFilerAddress validates that the filer address is safe to use
|
||||
func (h *FileBrowserHandlers) validateFilerAddress(address string) error {
|
||||
if address == "" {
|
||||
return fmt.Errorf("filer address cannot be empty")
|
||||
}
|
||||
|
||||
// CRITICAL: Only allow the configured filer address to prevent SSRF
|
||||
configuredFiler := h.adminServer.GetFilerAddress()
|
||||
normalizedAddress := pb.ServerAddress(address).ToHttpAddress()
|
||||
normalizedConfigured := pb.ServerAddress(configuredFiler).ToHttpAddress()
|
||||
if normalizedAddress != normalizedConfigured {
|
||||
return fmt.Errorf("address does not match configured filer: got %s, expected %s", address, configuredFiler)
|
||||
}
|
||||
|
||||
// Parse the normalized HTTP address to validate it's a proper host:port format.
|
||||
host, port, err := net.SplitHostPort(normalizedAddress)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid address format: %w", err)
|
||||
}
|
||||
|
||||
// Validate host is not empty
|
||||
if host == "" {
|
||||
return fmt.Errorf("host cannot be empty")
|
||||
}
|
||||
|
||||
// Validate port is numeric and in valid range
|
||||
if port == "" {
|
||||
return fmt.Errorf("port cannot be empty")
|
||||
}
|
||||
|
||||
portNum, err := strconv.Atoi(port)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid port number: %w", err)
|
||||
}
|
||||
|
||||
if portNum < 1 || portNum > 65535 {
|
||||
return fmt.Errorf("port number must be between 1 and 65535")
|
||||
}
|
||||
|
||||
return nil
|
||||
return h.uploadFileGrpc(ctx, filePath, fileHeader.Filename, fileHeader.Header.Get("Content-Type"), file)
|
||||
}
|
||||
|
||||
// validateAndCleanFilePath validates and cleans the file path to prevent path traversal
|
||||
@@ -507,161 +373,56 @@ func (h *FileBrowserHandlers) validateAndCleanFilePath(filePath string) (string,
|
||||
return cleanPath, nil
|
||||
}
|
||||
|
||||
// filerFileURL joins the filer HTTP address with a validated file path, URL-escaping
|
||||
// the path so that control characters and other bytes that are legal in S3 object keys
|
||||
// cannot inject into the HTTP request target.
|
||||
func filerFileURL(filerHttpAddress, cleanFilePath string) string {
|
||||
return filerHttpAddress + (&url.URL{Path: cleanFilePath}).EscapedPath()
|
||||
}
|
||||
|
||||
// fetchFileContent fetches file content from the filer and returns the content or an error.
|
||||
// fetchFileContent fetches file content via the filer gRPC service. It is
|
||||
// used for the "view as text" path, so the maxBytes cap matches the 1 MB
|
||||
// limit the caller already applies before invoking us.
|
||||
func (h *FileBrowserHandlers) fetchFileContent(filePath string, timeout time.Duration) (string, error) {
|
||||
filerAddress := h.adminServer.GetFilerAddress()
|
||||
if filerAddress == "" {
|
||||
return "", fmt.Errorf("filer address not configured")
|
||||
}
|
||||
|
||||
if err := h.validateFilerAddress(filerAddress); err != nil {
|
||||
return "", fmt.Errorf("invalid filer address configuration: %w", err)
|
||||
}
|
||||
filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress()
|
||||
|
||||
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Create the file URL with proper scheme based on TLS configuration
|
||||
fileURL := filerFileURL(filerHttpAddress, cleanFilePath)
|
||||
fileURL, err = h.httpClient.NormalizeHttpScheme(fileURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to construct file URL: %w", err)
|
||||
}
|
||||
|
||||
// lgtm[go/ssrf]
|
||||
// Safe: filerAddress validated by validateFilerAddress() to match configured filer
|
||||
// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
|
||||
client := h.newClientWithTimeout(timeout)
|
||||
req, err := http.NewRequest("GET", fileURL, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
h.addFilerJwtAuthHeader(req)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to fetch file from filer: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("filer returned status %d but failed to read response body: %w", resp.StatusCode, err)
|
||||
}
|
||||
return "", fmt.Errorf("filer returned status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
contentBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read file content: %w", err)
|
||||
}
|
||||
|
||||
return string(contentBytes), nil
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
return h.fetchFileContentGrpc(ctx, filePath, 0)
|
||||
}
|
||||
|
||||
// DownloadFile handles file download requests by proxying through the Admin UI server
|
||||
// This ensures mTLS works correctly since the Admin UI server has the client certificates
|
||||
// DownloadFile streams a file straight from the volume servers via the filer
|
||||
// gRPC service, so the admin file browser keeps working even when the filer
|
||||
// is started with -disableHttp=true.
|
||||
func (h *FileBrowserHandlers) DownloadFile(w http.ResponseWriter, r *http.Request) {
|
||||
filePath := r.URL.Query().Get("path")
|
||||
if filePath == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "File path is required")
|
||||
return
|
||||
}
|
||||
|
||||
// Get filer address
|
||||
filerAddress := h.adminServer.GetFilerAddress()
|
||||
if filerAddress == "" {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Filer address not configured")
|
||||
return
|
||||
}
|
||||
|
||||
// Validate filer address to prevent SSRF
|
||||
if err := h.validateFilerAddress(filerAddress); err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Invalid filer address configuration")
|
||||
return
|
||||
}
|
||||
filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress()
|
||||
|
||||
// Validate and sanitize the file path
|
||||
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid file path: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Create the download URL with proper scheme based on TLS configuration
|
||||
downloadURL := filerFileURL(filerHttpAddress, cleanFilePath)
|
||||
downloadURL, err = h.httpClient.NormalizeHttpScheme(downloadURL)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to construct download URL: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Proxy the download through the Admin UI server to support mTLS
|
||||
// lgtm[go/ssrf]
|
||||
// Safe: filerAddress validated by validateFilerAddress() to match configured filer
|
||||
// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
|
||||
// Use request context so download is cancelled when client disconnects
|
||||
req, err := http.NewRequestWithContext(r.Context(), "GET", downloadURL, nil)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to create request: "+err.Error())
|
||||
return
|
||||
}
|
||||
client := h.newClientWithTimeout(5 * time.Minute) // Longer timeout for large file downloads
|
||||
|
||||
h.addFilerJwtAuthHeader(req)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusBadGateway, "Failed to fetch file from filer: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
writeJSONError(w, resp.StatusCode, fmt.Sprintf("Filer returned status %d but failed to read response body: %v", resp.StatusCode, err))
|
||||
tracker := &responseWriteTracker{ResponseWriter: w}
|
||||
if err := h.downloadFileGrpc(r.Context(), filePath, tracker); err != nil {
|
||||
// Once bytes have been written we can't switch to a JSON error body
|
||||
// without corrupting the partial response — log and stop. Before any
|
||||
// write the response is still uncommitted, so a 502 with details is
|
||||
// safe.
|
||||
if tracker.committed {
|
||||
glog.Errorf("Error streaming file download: %v", err)
|
||||
return
|
||||
}
|
||||
writeJSONError(w, resp.StatusCode, fmt.Sprintf("Filer returned status %d: %s", resp.StatusCode, string(body)))
|
||||
return
|
||||
writeJSONError(w, http.StatusBadGateway, "Failed to fetch file: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// Set headers for file download
|
||||
fileName := filepath.Base(cleanFilePath)
|
||||
// Use mime.FormatMediaType for RFC 6266 compliant Content-Disposition,
|
||||
// properly handling non-ASCII characters and special characters
|
||||
w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": fileName}))
|
||||
// responseWriteTracker wraps http.ResponseWriter to record whether the
|
||||
// response has been committed (status line + headers sent). DownloadFile
|
||||
// uses this instead of probing Header() so future header-setting code
|
||||
// reorganization can't silently break the "did we already send bytes?"
|
||||
// detection.
|
||||
type responseWriteTracker struct {
|
||||
http.ResponseWriter
|
||||
committed bool
|
||||
}
|
||||
|
||||
// Use content type from filer response, or default to octet-stream
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
func (t *responseWriteTracker) WriteHeader(code int) {
|
||||
t.committed = true
|
||||
t.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
// Set content length if available
|
||||
if resp.ContentLength > 0 {
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", resp.ContentLength))
|
||||
}
|
||||
|
||||
// Stream the response body to the client
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, err = io.Copy(w, resp.Body)
|
||||
if err != nil {
|
||||
glog.Errorf("Error streaming file download: %v", err)
|
||||
}
|
||||
func (t *responseWriteTracker) Write(p []byte) (int, error) {
|
||||
t.committed = true
|
||||
return t.ResponseWriter.Write(p)
|
||||
}
|
||||
|
||||
// ViewFile handles file viewing requests (for text files, images, etc.)
|
||||
@@ -883,64 +644,16 @@ func (h *FileBrowserHandlers) formatBytes(bytes int64) string {
|
||||
|
||||
// Helper function to check if a file is likely a text file by checking content
|
||||
func (h *FileBrowserHandlers) isLikelyTextFile(filePath string, maxCheckSize int64) bool {
|
||||
filerAddress := h.adminServer.GetFilerAddress()
|
||||
if filerAddress == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
// Validate filer address to prevent SSRF
|
||||
if err := h.validateFilerAddress(filerAddress); err != nil {
|
||||
glog.Errorf("Invalid filer address: %v", err)
|
||||
return false
|
||||
}
|
||||
filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress()
|
||||
|
||||
cleanFilePath, err := h.validateAndCleanFilePath(filePath)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
content, err := h.fetchFileContentGrpc(ctx, filePath, int(maxCheckSize))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// Create the file URL with proper scheme based on TLS configuration
|
||||
fileURL := filerFileURL(filerHttpAddress, cleanFilePath)
|
||||
fileURL, err = h.httpClient.NormalizeHttpScheme(fileURL)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to normalize URL scheme: %v", err)
|
||||
return false
|
||||
if len(content) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
// lgtm[go/ssrf]
|
||||
// Safe: filerAddress validated by validateFilerAddress() to match configured filer
|
||||
// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
|
||||
client := h.newClientWithTimeout(10 * time.Second)
|
||||
req, err := http.NewRequest("GET", fileURL, nil)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to create request: %v", err)
|
||||
return false
|
||||
}
|
||||
h.addFilerJwtAuthHeader(req)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return false
|
||||
}
|
||||
|
||||
// Read first few bytes to check if it's text
|
||||
buffer := make([]byte, min(maxCheckSize, 512))
|
||||
n, err := resp.Body.Read(buffer)
|
||||
if err != nil && err != io.EOF {
|
||||
return false
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
return true // Empty file can be considered text
|
||||
}
|
||||
|
||||
// Check if content is printable text
|
||||
return h.isPrintableText(buffer[:n])
|
||||
return h.isPrintableText([]byte(content))
|
||||
}
|
||||
|
||||
// Helper function to check if content is printable text
|
||||
@@ -973,35 +686,3 @@ func min(a, b int64) int64 {
|
||||
return b
|
||||
}
|
||||
|
||||
// setupFilerJwtAuth generates a JWT token and adds it to the request Authorization header if configured.
|
||||
func (h *FileBrowserHandlers) setupFilerJwtAuth(req *http.Request, keyPath, expiresPath, operation string) {
|
||||
// Load security configuration
|
||||
v := util.GetViper()
|
||||
|
||||
// Read Filer JWT token from security.toml
|
||||
signingKey := security.SigningKey(v.GetString(keyPath))
|
||||
expiresAfterSec := v.GetInt(expiresPath)
|
||||
|
||||
// Generate JWT token to authenticate with Filer
|
||||
var jwtToken security.EncodedJwt
|
||||
if len(signingKey) > 0 {
|
||||
jwtToken = security.GenJwtForFilerServer(signingKey, expiresAfterSec)
|
||||
glog.V(4).Infof("Generated JWT token for %s (expires in %d sec)", operation, expiresAfterSec)
|
||||
} else {
|
||||
if v.GetString("jwt.signing.key") != "" {
|
||||
glog.Warningf("JWT %s key not configured, but general JWT security is enabled. %s without authentication.", keyPath, operation)
|
||||
} else {
|
||||
glog.V(1).Infof("No JWT signing key configured, %s without authentication", operation)
|
||||
}
|
||||
}
|
||||
|
||||
// Add JWT Token to Authorization Header
|
||||
if jwtToken != "" {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", string(jwtToken)))
|
||||
glog.V(4).Infof("Added JWT authorization header for %s", operation)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *FileBrowserHandlers) addFilerJwtAuthHeader(req *http.Request) {
|
||||
h.setupFilerJwtAuth(req, "jwt.filer_signing.read.key", "jwt.filer_signing.read.expires_after_seconds", "filer request")
|
||||
}
|
||||
@@ -42,21 +42,3 @@ func TestValidateAndCleanFilePath_RejectsEmpty(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilerFileURL_EscapesControlChars(t *testing.T) {
|
||||
cases := []struct {
|
||||
addr string
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{"http://127.0.0.1:8888", "/buckets/profilebuilder/3testGB.zip\n ", "http://127.0.0.1:8888/buckets/profilebuilder/3testGB.zip%0A%20"},
|
||||
{"http://127.0.0.1:8888", "/buckets/profilebuilder/file\rname", "http://127.0.0.1:8888/buckets/profilebuilder/file%0Dname"},
|
||||
{"http://127.0.0.1:8888", "/buckets/profilebuilder/file\x00name", "http://127.0.0.1:8888/buckets/profilebuilder/file%00name"},
|
||||
// Plain path round-trips unchanged.
|
||||
{"http://h:1", "/a/b.txt", "http://h:1/a/b.txt"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := filerFileURL(tc.addr, tc.path); got != tc.want {
|
||||
t.Errorf("filerFileURL(%q, %q) = %q, want %q", tc.addr, tc.path, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user