filer: tighten the format HTTP surface

- namespace the query parameters as format.ingest, format.repack and
  format.view, following the mv.from/cp.from dotted convention, so the
  general endpoints cannot collide with pass-through client parameters;
  requests naming both ingest and repack are rejected
- state Accept-Ranges: none on view responses, which always answer with
  whole documents or whole extents
- derive the small-content permission from the boundary source instead
  of a second positional bool that a call site could silently swap
- validate the hls-ts layout before returning it, making the formattest
  invariant enforced rather than emergent

weed/format/format.go

@@ -18,6 +18,10 @@ import (
 // x-seaweedfs- prefix keeps it out of HTTP response headers.
 const LayoutKey = "x-seaweedfs-format-layout"
 
+// ViewParam is the query parameter selecting a format view on GET requests.
+// Adapters rendering self-referential URLs must use it.
+const ViewParam = "format.view"
+
 // ErrNoSuchView reports that a view request addresses nothing servable; the
 // server answers 404.
 var ErrNoSuchView = errors.New("no such view")

weed/format/hlsts/hlsts.go

@@ -227,12 +227,17 @@ func (Adapter) IndexSidecar(sidecar []byte) (*format.Layout, error) {
 		return nil, fmt.Errorf("EXT-X-TARGETDURATION %d is smaller than the longest segment duration %d", info.TargetDuration, minimumTarget)
 	}
 
-	return &format.Layout{
+	layout := &format.Layout{
 		Format:      FormatName,
 		ExtentSizes: sizes,
 		Align:       TSPacketSize,
 		Payload:     info.encode(),
-	}, nil
+	}
+	// valid by construction, but enforce the formattest invariant explicitly
+	if err := layout.Validate(-1); err != nil {
+		return nil, err
+	}
+	return layout, nil
 }
 
 // View serves the generated playlist, or maps ?seq=N to its extent.
@@ -265,7 +270,7 @@ func renderPlaylist(name string, info *playlistInfo) []byte {
 	escapedName := url.PathEscape(name)
 	for i, durationMs := range info.DurationsMs {
 		fmt.Fprintf(&out, "#EXTINF:%.3f,\n", float64(durationMs)/1000)
-		fmt.Fprintf(&out, "%s?view=%s&seq=%d\n", escapedName, FormatName, int64(i)+info.MediaSequence)
+		fmt.Fprintf(&out, "%s?%s=%s&seq=%d\n", escapedName, format.ViewParam, FormatName, int64(i)+info.MediaSequence)
 	}
 	out.WriteString("#EXT-X-ENDLIST\n")
 	return []byte(out.String())

weed/format/hlsts/hlsts_test.go

@@ -121,11 +121,11 @@ func TestViewPlaylist(t *testing.T) {
 #EXT-X-MEDIA-SEQUENCE:0
 #EXT-X-PLAYLIST-TYPE:VOD
 #EXTINF:6.000,
-movie.ts?view=hls-ts&seq=0
+movie.ts?format.view=hls-ts&seq=0
 #EXTINF:6.000,
-movie.ts?view=hls-ts&seq=1
+movie.ts?format.view=hls-ts&seq=1
 #EXTINF:2.500,
-movie.ts?view=hls-ts&seq=2
+movie.ts?format.view=hls-ts&seq=2
 #EXT-X-ENDLIST
 `
 	if string(plan.Body) != want {

weed/server/filer_server_format.go

@@ -26,6 +26,11 @@ import (
 )
 
 const (
+	// Query parameters follow the mv.from/cp.from dotted convention so the
+	// general POST endpoint cannot collide with pass-through client params.
+	formatIngestParam = "format.ingest"
+	formatRepackParam = "format.repack"
+
 	maxFormatSidecarBytes = 16 << 20
 	formatSniffBytes      = 512
 	// defaultFormatChunkSizeMB caps extent chunks when no maxMB is configured.
@@ -60,11 +65,11 @@ func copyStandardHeadersToExtended(r *http.Request, extended map[string][]byte)
 	}
 }
 
-// formatIngest handles POST /path?format=<adapter>: a multipart body with an
-// "index" sidecar part describing the media's extents, then the "media" bytes.
-// Storage chunks are cut on the extent boundaries the sidecar declares.
+// formatIngest handles POST /path?format.ingest=<adapter>: a multipart body
+// with an "index" sidecar part describing the media's extents, then the
+// "media" bytes. Storage chunks are cut on the boundaries the sidecar declares.
 func (fs *FilerServer) formatIngest(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) {
-	adapterName := r.URL.Query().Get("format")
+	adapterName := r.URL.Query().Get(formatIngestParam)
 	adapter := format.ByName(adapterName)
 	if adapter == nil {
 		writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("unknown format %q", adapterName))
@@ -137,7 +142,7 @@ func (fs *FilerServer) formatIngest(ctx context.Context, w http.ResponseWriter,
 	}
 
 	cutter := layout.Cutter(fs.formatChunkSizeLimit(r))
-	fileChunks, md5Hash, written, uploadErr, _ := fs.uploadReaderToBoundedChunks(ctx, r, mediaPart, 0, cutter, false, path.Base(r.URL.Path), contentType, false, so)
+	fileChunks, md5Hash, written, uploadErr, _ := fs.uploadReaderToBoundedChunks(ctx, r, mediaPart, 0, cutter, path.Base(r.URL.Path), contentType, false, so)
 	cleanup := func() { fs.filer.DeleteUncommittedChunks(context.WithoutCancel(ctx), fileChunks) }
 	if uploadErr != nil {
 		cleanup()
@@ -204,11 +209,11 @@ func (fs *FilerServer) formatIngest(ctx context.Context, w http.ResponseWriter,
 	writeJsonQuiet(w, r, http.StatusCreated, FilerPostResult{Name: entry.Name(), Size: written})
 }
 
-// formatRepack handles POST /path?repack=<adapter>: it derives the layout from
-// the stored bytes and rewrites the entry's chunks cut on extent boundaries.
-// The bytes do not change, only where they are cut.
+// formatRepack handles POST /path?format.repack=<adapter>: it derives the
+// layout from the stored bytes and rewrites the entry's chunks cut on extent
+// boundaries. The bytes do not change, only where they are cut.
 func (fs *FilerServer) formatRepack(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) {
-	adapterName := r.URL.Query().Get("repack")
+	adapterName := r.URL.Query().Get(formatRepackParam)
 	adapter := format.ByName(adapterName)
 	if adapter == nil {
 		writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("unknown format %q", adapterName))
@@ -306,7 +311,7 @@ func (fs *FilerServer) formatRepack(ctx context.Context, w http.ResponseWriter,
 	}
 
 	cutter := layout.Cutter(fs.formatChunkSizeLimit(r))
-	newChunks, md5Hash, written, uploadErr, _ := fs.uploadReaderToBoundedChunks(ctx, r, io.NewSectionReader(readerAt, 0, size), 0, cutter, false, entry.Name(), entry.Attr.Mime, false, so)
+	newChunks, md5Hash, written, uploadErr, _ := fs.uploadReaderToBoundedChunks(ctx, r, io.NewSectionReader(readerAt, 0, size), 0, cutter, entry.Name(), entry.Attr.Mime, false, so)
 	cleanup := func() { fs.filer.DeleteUncommittedChunks(context.WithoutCancel(ctx), newChunks) }
 	if uploadErr != nil {
 		cleanup()
@@ -402,6 +407,8 @@ func (fs *FilerServer) serveFormatView(ctx context.Context, w http.ResponseWrite
 			w.Header().Set(k, string(v))
 		}
 	}
+	// view responses are whole documents or whole extents
+	w.Header().Set("Accept-Ranges", "none")
 	w.Header().Set("Content-Type", plan.ContentType)
 	SetEtag(w, filer.ETagEntry(entry))
 

weed/server/filer_server_handlers_read.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/format"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
@@ -139,7 +140,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	if viewName := query.Get("view"); viewName != "" {
+	if viewName := query.Get(format.ViewParam); viewName != "" {
 		fs.serveFormatView(ctx, w, r, entry, viewName)
 		return
 	}

weed/server/filer_server_handlers_write.go

@@ -136,9 +136,13 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, conte
 		fs.move(ctx, w, r, so)
 	} else if query.Has("cp.from") {
 		fs.copy(ctx, w, r, so)
-	} else if query.Get("format") != "" {
+	} else if query.Get(formatIngestParam) != "" {
+		if query.Get(formatRepackParam) != "" {
+			writeJsonError(w, r, http.StatusBadRequest, errors.New(formatIngestParam+" and "+formatRepackParam+" are mutually exclusive"))
+			return
+		}
 		fs.formatIngest(ctx, w, r, so)
-	} else if query.Get("repack") != "" {
+	} else if query.Get(formatRepackParam) != "" {
 		fs.formatRepack(ctx, w, r, so)
 	} else {
 		fs.autoChunk(ctx, w, r, contentLength, so)

weed/server/filer_server_handlers_write_upload.go

@@ -31,8 +31,9 @@ var bufPool = sync.Pool{
 
 // ChunkBoundaries decides where the upload loop may cut a storage chunk.
 type ChunkBoundaries interface {
-	// NextChunkSize returns the size of the chunk starting at offset, or 0 when
-	// no further chunks are expected.
+	// NextChunkSize returns the size of the chunk starting at the absolute
+	// file offset (startOffset included), or 0 when no further chunks are
+	// expected. Returned sizes are bounded: they size in-memory buffers.
 	NextChunkSize(offset int64) int64
 }
 
@@ -62,14 +63,15 @@ func (fs *FilerServer) uploadRequestToChunks(ctx context.Context, w http.Respons
 }
 
 func (fs *FilerServer) uploadReaderToChunks(ctx context.Context, r *http.Request, reader io.Reader, startOffset int64, chunkSize int32, fileName, contentType string, isAppend bool, so *operation.StorageOption) (fileChunks []*filer_pb.FileChunk, md5Hash hash.Hash, chunkOffset int64, uploadErr error, smallContent []byte) {
-	return fs.uploadReaderToBoundedChunks(ctx, r, reader, startOffset, fixedChunkSize(chunkSize), true, fileName, contentType, isAppend, so)
+	return fs.uploadReaderToBoundedChunks(ctx, r, reader, startOffset, fixedChunkSize(chunkSize), fileName, contentType, isAppend, so)
 }
 
-// uploadReaderToBoundedChunks cuts chunks where boundaries allows instead of at
-// a fixed size. allowInline permits the small-content optimization, which must
-// stay off when chunk boundaries carry meaning.
-func (fs *FilerServer) uploadReaderToBoundedChunks(ctx context.Context, r *http.Request, reader io.Reader, startOffset int64, boundaries ChunkBoundaries, allowInline bool, fileName, contentType string, isAppend bool, so *operation.StorageOption) (fileChunks []*filer_pb.FileChunk, md5Hash hash.Hash, chunkOffset int64, uploadErr error, smallContent []byte) {
+// uploadReaderToBoundedChunks cuts chunks where boundaries allows instead of
+// at a fixed size. The small-content optimization only applies to fixed-size
+// chunking: it must stay off when chunk boundaries carry meaning.
+func (fs *FilerServer) uploadReaderToBoundedChunks(ctx context.Context, r *http.Request, reader io.Reader, startOffset int64, boundaries ChunkBoundaries, fileName, contentType string, isAppend bool, so *operation.StorageOption) (fileChunks []*filer_pb.FileChunk, md5Hash hash.Hash, chunkOffset int64, uploadErr error, smallContent []byte) {
 
+	_, allowInline := boundaries.(fixedChunkSize)
 	md5Hash = md5.New()
 	chunkOffset = startOffset
 	var partReader = io.NopCloser(io.TeeReader(reader, md5Hash))