From 79297b549ee3c7902f0987227d3d0abcd24d8693 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 10 Aug 2026 00:59:38 -0700 Subject: [PATCH] filer: tighten the format HTTP surface - namespace the query parameters as format.ingest, format.repack and format.view, following the mv.from/cp.from dotted convention, so the general endpoints cannot collide with pass-through client parameters; requests naming both ingest and repack are rejected - state Accept-Ranges: none on view responses, which always answer with whole documents or whole extents - derive the small-content permission from the boundary source instead of a second positional bool that a call site could silently swap - validate the hls-ts layout before returning it, making the formattest invariant enforced rather than emergent --- weed/format/format.go | 4 +++ weed/format/hlsts/hlsts.go | 11 +++++--- weed/format/hlsts/hlsts_test.go | 6 ++--- weed/server/filer_server_format.go | 27 ++++++++++++------- weed/server/filer_server_handlers_read.go | 3 ++- weed/server/filer_server_handlers_write.go | 8 ++++-- .../filer_server_handlers_write_upload.go | 16 ++++++----- 7 files changed, 49 insertions(+), 26 deletions(-) diff --git a/weed/format/format.go b/weed/format/format.go index bc557a309..ccd16158a 100644 --- a/weed/format/format.go +++ b/weed/format/format.go @@ -18,6 +18,10 @@ import ( // x-seaweedfs- prefix keeps it out of HTTP response headers. const LayoutKey = "x-seaweedfs-format-layout" +// ViewParam is the query parameter selecting a format view on GET requests. +// Adapters rendering self-referential URLs must use it. +const ViewParam = "format.view" + // ErrNoSuchView reports that a view request addresses nothing servable; the // server answers 404. var ErrNoSuchView = errors.New("no such view") diff --git a/weed/format/hlsts/hlsts.go b/weed/format/hlsts/hlsts.go index 41972bac2..55b64fbdb 100644 --- a/weed/format/hlsts/hlsts.go +++ b/weed/format/hlsts/hlsts.go @@ -227,12 +227,17 @@ func (Adapter) IndexSidecar(sidecar []byte) (*format.Layout, error) { return nil, fmt.Errorf("EXT-X-TARGETDURATION %d is smaller than the longest segment duration %d", info.TargetDuration, minimumTarget) } - return &format.Layout{ + layout := &format.Layout{ Format: FormatName, ExtentSizes: sizes, Align: TSPacketSize, Payload: info.encode(), - }, nil + } + // valid by construction, but enforce the formattest invariant explicitly + if err := layout.Validate(-1); err != nil { + return nil, err + } + return layout, nil } // View serves the generated playlist, or maps ?seq=N to its extent. @@ -265,7 +270,7 @@ func renderPlaylist(name string, info *playlistInfo) []byte { escapedName := url.PathEscape(name) for i, durationMs := range info.DurationsMs { fmt.Fprintf(&out, "#EXTINF:%.3f,\n", float64(durationMs)/1000) - fmt.Fprintf(&out, "%s?view=%s&seq=%d\n", escapedName, FormatName, int64(i)+info.MediaSequence) + fmt.Fprintf(&out, "%s?%s=%s&seq=%d\n", escapedName, format.ViewParam, FormatName, int64(i)+info.MediaSequence) } out.WriteString("#EXT-X-ENDLIST\n") return []byte(out.String()) diff --git a/weed/format/hlsts/hlsts_test.go b/weed/format/hlsts/hlsts_test.go index ceeb3bf0f..dfa2cf794 100644 --- a/weed/format/hlsts/hlsts_test.go +++ b/weed/format/hlsts/hlsts_test.go @@ -121,11 +121,11 @@ func TestViewPlaylist(t *testing.T) { #EXT-X-MEDIA-SEQUENCE:0 #EXT-X-PLAYLIST-TYPE:VOD #EXTINF:6.000, -movie.ts?view=hls-ts&seq=0 +movie.ts?format.view=hls-ts&seq=0 #EXTINF:6.000, -movie.ts?view=hls-ts&seq=1 +movie.ts?format.view=hls-ts&seq=1 #EXTINF:2.500, -movie.ts?view=hls-ts&seq=2 +movie.ts?format.view=hls-ts&seq=2 #EXT-X-ENDLIST ` if string(plan.Body) != want { diff --git a/weed/server/filer_server_format.go b/weed/server/filer_server_format.go index b89c153a1..fa2211c1e 100644 --- a/weed/server/filer_server_format.go +++ b/weed/server/filer_server_format.go @@ -26,6 +26,11 @@ import ( ) const ( + // Query parameters follow the mv.from/cp.from dotted convention so the + // general POST endpoint cannot collide with pass-through client params. + formatIngestParam = "format.ingest" + formatRepackParam = "format.repack" + maxFormatSidecarBytes = 16 << 20 formatSniffBytes = 512 // defaultFormatChunkSizeMB caps extent chunks when no maxMB is configured. @@ -60,11 +65,11 @@ func copyStandardHeadersToExtended(r *http.Request, extended map[string][]byte) } } -// formatIngest handles POST /path?format=: a multipart body with an -// "index" sidecar part describing the media's extents, then the "media" bytes. -// Storage chunks are cut on the extent boundaries the sidecar declares. +// formatIngest handles POST /path?format.ingest=: a multipart body +// with an "index" sidecar part describing the media's extents, then the +// "media" bytes. Storage chunks are cut on the boundaries the sidecar declares. func (fs *FilerServer) formatIngest(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) { - adapterName := r.URL.Query().Get("format") + adapterName := r.URL.Query().Get(formatIngestParam) adapter := format.ByName(adapterName) if adapter == nil { writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("unknown format %q", adapterName)) @@ -137,7 +142,7 @@ func (fs *FilerServer) formatIngest(ctx context.Context, w http.ResponseWriter, } cutter := layout.Cutter(fs.formatChunkSizeLimit(r)) - fileChunks, md5Hash, written, uploadErr, _ := fs.uploadReaderToBoundedChunks(ctx, r, mediaPart, 0, cutter, false, path.Base(r.URL.Path), contentType, false, so) + fileChunks, md5Hash, written, uploadErr, _ := fs.uploadReaderToBoundedChunks(ctx, r, mediaPart, 0, cutter, path.Base(r.URL.Path), contentType, false, so) cleanup := func() { fs.filer.DeleteUncommittedChunks(context.WithoutCancel(ctx), fileChunks) } if uploadErr != nil { cleanup() @@ -204,11 +209,11 @@ func (fs *FilerServer) formatIngest(ctx context.Context, w http.ResponseWriter, writeJsonQuiet(w, r, http.StatusCreated, FilerPostResult{Name: entry.Name(), Size: written}) } -// formatRepack handles POST /path?repack=: it derives the layout from -// the stored bytes and rewrites the entry's chunks cut on extent boundaries. -// The bytes do not change, only where they are cut. +// formatRepack handles POST /path?format.repack=: it derives the +// layout from the stored bytes and rewrites the entry's chunks cut on extent +// boundaries. The bytes do not change, only where they are cut. func (fs *FilerServer) formatRepack(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) { - adapterName := r.URL.Query().Get("repack") + adapterName := r.URL.Query().Get(formatRepackParam) adapter := format.ByName(adapterName) if adapter == nil { writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("unknown format %q", adapterName)) @@ -306,7 +311,7 @@ func (fs *FilerServer) formatRepack(ctx context.Context, w http.ResponseWriter, } cutter := layout.Cutter(fs.formatChunkSizeLimit(r)) - newChunks, md5Hash, written, uploadErr, _ := fs.uploadReaderToBoundedChunks(ctx, r, io.NewSectionReader(readerAt, 0, size), 0, cutter, false, entry.Name(), entry.Attr.Mime, false, so) + newChunks, md5Hash, written, uploadErr, _ := fs.uploadReaderToBoundedChunks(ctx, r, io.NewSectionReader(readerAt, 0, size), 0, cutter, entry.Name(), entry.Attr.Mime, false, so) cleanup := func() { fs.filer.DeleteUncommittedChunks(context.WithoutCancel(ctx), newChunks) } if uploadErr != nil { cleanup() @@ -402,6 +407,8 @@ func (fs *FilerServer) serveFormatView(ctx context.Context, w http.ResponseWrite w.Header().Set(k, string(v)) } } + // view responses are whole documents or whole extents + w.Header().Set("Accept-Ranges", "none") w.Header().Set("Content-Type", plan.ContentType) SetEtag(w, filer.ETagEntry(entry)) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 720ec3344..1eb5debc1 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -14,6 +14,7 @@ import ( "time" "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/format" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" @@ -139,7 +140,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) return } - if viewName := query.Get("view"); viewName != "" { + if viewName := query.Get(format.ViewParam); viewName != "" { fs.serveFormatView(ctx, w, r, entry, viewName) return } diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 809205072..e3b1663e2 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -136,9 +136,13 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, conte fs.move(ctx, w, r, so) } else if query.Has("cp.from") { fs.copy(ctx, w, r, so) - } else if query.Get("format") != "" { + } else if query.Get(formatIngestParam) != "" { + if query.Get(formatRepackParam) != "" { + writeJsonError(w, r, http.StatusBadRequest, errors.New(formatIngestParam+" and "+formatRepackParam+" are mutually exclusive")) + return + } fs.formatIngest(ctx, w, r, so) - } else if query.Get("repack") != "" { + } else if query.Get(formatRepackParam) != "" { fs.formatRepack(ctx, w, r, so) } else { fs.autoChunk(ctx, w, r, contentLength, so) diff --git a/weed/server/filer_server_handlers_write_upload.go b/weed/server/filer_server_handlers_write_upload.go index ed9d37d2a..3799e69b4 100644 --- a/weed/server/filer_server_handlers_write_upload.go +++ b/weed/server/filer_server_handlers_write_upload.go @@ -31,8 +31,9 @@ var bufPool = sync.Pool{ // ChunkBoundaries decides where the upload loop may cut a storage chunk. type ChunkBoundaries interface { - // NextChunkSize returns the size of the chunk starting at offset, or 0 when - // no further chunks are expected. + // NextChunkSize returns the size of the chunk starting at the absolute + // file offset (startOffset included), or 0 when no further chunks are + // expected. Returned sizes are bounded: they size in-memory buffers. NextChunkSize(offset int64) int64 } @@ -62,14 +63,15 @@ func (fs *FilerServer) uploadRequestToChunks(ctx context.Context, w http.Respons } func (fs *FilerServer) uploadReaderToChunks(ctx context.Context, r *http.Request, reader io.Reader, startOffset int64, chunkSize int32, fileName, contentType string, isAppend bool, so *operation.StorageOption) (fileChunks []*filer_pb.FileChunk, md5Hash hash.Hash, chunkOffset int64, uploadErr error, smallContent []byte) { - return fs.uploadReaderToBoundedChunks(ctx, r, reader, startOffset, fixedChunkSize(chunkSize), true, fileName, contentType, isAppend, so) + return fs.uploadReaderToBoundedChunks(ctx, r, reader, startOffset, fixedChunkSize(chunkSize), fileName, contentType, isAppend, so) } -// uploadReaderToBoundedChunks cuts chunks where boundaries allows instead of at -// a fixed size. allowInline permits the small-content optimization, which must -// stay off when chunk boundaries carry meaning. -func (fs *FilerServer) uploadReaderToBoundedChunks(ctx context.Context, r *http.Request, reader io.Reader, startOffset int64, boundaries ChunkBoundaries, allowInline bool, fileName, contentType string, isAppend bool, so *operation.StorageOption) (fileChunks []*filer_pb.FileChunk, md5Hash hash.Hash, chunkOffset int64, uploadErr error, smallContent []byte) { +// uploadReaderToBoundedChunks cuts chunks where boundaries allows instead of +// at a fixed size. The small-content optimization only applies to fixed-size +// chunking: it must stay off when chunk boundaries carry meaning. +func (fs *FilerServer) uploadReaderToBoundedChunks(ctx context.Context, r *http.Request, reader io.Reader, startOffset int64, boundaries ChunkBoundaries, fileName, contentType string, isAppend bool, so *operation.StorageOption) (fileChunks []*filer_pb.FileChunk, md5Hash hash.Hash, chunkOffset int64, uploadErr error, smallContent []byte) { + _, allowInline := boundaries.(fixedChunkSize) md5Hash = md5.New() chunkOffset = startOffset var partReader = io.NopCloser(io.TeeReader(reader, md5Hash))