operation: probe octet-stream uploads for gzip like untyped ones (#12584)

* operation: probe octet-stream uploads for gzip like untyped ones

The 128-byte gzip probe ran only when the caller passed no MimeType, so an
upload labelled application/octet-stream or binary/octet-stream was never
compressed. S3 PutObject and UploadPart pass the request Content-Type (or the
octet-stream that mimeDetect fills in), so every binary S3 upload skipped
compression that the same bytes get through the filer.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* operation: check the probed upload keeps the caller's content type

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

weed/operation/needle_parse_test.go

@@ -6,7 +6,10 @@ import (
 	"fmt"
 	"io"
 	"math/rand"
+	"mime"
+	"mime/multipart"
 	"net/http"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -146,6 +149,68 @@ func TestUploadKeepsMarginalGzipUncompressed(t *testing.T) {
 	assert.Equal(t, data, stored.Data)
 }
 
+type storingClient struct {
+	stored      *needle.Needle
+	contentType string
+}
+
+func (c *storingClient) Do(req *http.Request) (*http.Response, error) {
+	body, err := io.ReadAll(req.Body)
+	if err != nil {
+		return nil, err
+	}
+	_, params, _ := mime.ParseMediaType(req.Header.Get("Content-Type"))
+	part, err := multipart.NewReader(bytes.NewReader(body), params["boundary"]).NextPart()
+	if err != nil {
+		return nil, err
+	}
+	c.contentType = part.Header.Get("Content-Type")
+	req.Body = io.NopCloser(bytes.NewReader(body))
+	n, _, _, err := needle.CreateNeedleFromRequest(req, false, 1024*1024, &bytes.Buffer{})
+	if err != nil {
+		return nil, err
+	}
+	c.stored = n
+	return &http.Response{StatusCode: http.StatusCreated, Body: io.NopCloser(strings.NewReader("{}"))}, nil
+}
+
+func TestUploadProbesGenericBinaryMimeTypes(t *testing.T) {
+	zeros := make([]byte, 64*1024)
+	random := make([]byte, 64*1024)
+	rand.New(rand.NewSource(1)).Read(random)
+	for _, tc := range []struct {
+		mimeType string
+		data     []byte
+		gzipped  bool
+	}{
+		{"", zeros, true},
+		{"application/octet-stream", zeros, true},
+		{"binary/octet-stream", zeros, true},
+		{"Application/Octet-Stream; charset=binary", zeros, true},
+		{"application/octet-stream", random, false},
+		{"video/mp4", zeros, false},
+	} {
+		client := &storingClient{}
+		result, err := newUploader(client).UploadData(context.Background(), tc.data, &UploadOption{
+			UploadUrl: "http://localhost:8080/389,0f084d17353afda0",
+			MimeType:  tc.mimeType,
+		})
+		if err != nil || client.stored == nil {
+			t.Fatalf("%q: upload: %v", tc.mimeType, err)
+		}
+		assert.Equal(t, tc.gzipped, client.stored.IsCompressed(), "mime %q", tc.mimeType)
+		assert.Equal(t, tc.gzipped, result.Gzip > 0, "mime %q", tc.mimeType)
+		if tc.mimeType != "" {
+			assert.Equal(t, tc.mimeType, client.contentType)
+		}
+		if tc.gzipped {
+			unzipped, err := util.DecompressData(client.stored.Data)
+			assert.NoError(t, err)
+			assert.Equal(t, tc.data, unzipped)
+		}
+	}
+}
+
 func TestReplicationKeepsTheSourceMime(t *testing.T) {
 	for _, mimeType := range []string{"", "application/octet-stream", "Text/Plain; charset=UTF-8"} {
 		mockClient := &MockClient{}

weed/operation/upload_content.go

@@ -379,6 +379,17 @@ func (uploader *Uploader) retriedUploadData(ctx context.Context, data []byte, op
 	return
 }
 
+// isGenericBinaryMimeType reports a content type that says nothing about the bytes,
+// such as the application/octet-stream most S3 clients send for unknown files.
+func isGenericBinaryMimeType(mimeType string) bool {
+	mediaType, _, _ := strings.Cut(mimeType, ";")
+	switch strings.ToLower(strings.TrimSpace(mediaType)) {
+	case "", "application/octet-stream", "binary/octet-stream":
+		return true
+	}
+	return false
+}
+
 func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
 	contentIsGzipped := option.IsInputCompressed
 	shouldGzipNow := false
@@ -392,7 +403,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
 		}
 		if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Ext(option.Filename), option.MimeType); iAmSure && shouldBeCompressed {
 			shouldGzipNow = true
-		} else if !iAmSure && option.MimeType == "" && len(data) > 16*1024 {
+		} else if !iAmSure && isGenericBinaryMimeType(option.MimeType) && len(data) > 16*1024 {
 			var compressed []byte
 			compressed, err = util.GzipData(data[0:128])
 			if err != nil {