compression: compress JSON uploads, and match file types by extension (#12339)

* compression: compress JSON uploads, and match file types by extension

IsCompressableFileType did not recognise JSON media types, so S3 uploads of application/json (which carry no file name) were never compressed. Its callers also passed filepath.Base of the file name where it expects the extension, so the .json/.js/.html/... extension rules never matched a real file name. Both callers now pass filepath.Ext; the function lower-cases the extension and media type, ignores media-type parameters ("application/json; charset=utf-8"), and treats */json and *+json types as compressible.

* compression: treat application/json-seq as JSON

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* volume(rust): compress JSON uploads and match media types without parameters

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* operation: keep uploader gzip only when it saves at least 10%

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Pat White <5015863+patwhite@users.noreply.github.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

seaweed-volume/src/server/handlers.rs

@@ -3101,15 +3101,12 @@ pub async fn post_handler(
 
     // Auto-compress compressible file types (matches Go's IsCompressableFileType).
     // Only compress if not already gzipped and compression saves >10%.
-    // Go uses filepath.Base(pu.FileName) for extension detection (not the URL path).
     let (final_data, final_is_gzipped) = if !is_gzipped && !is_chunk_manifest {
-        let ext = {
-            let dot_pos = filename.rfind('.');
-            dot_pos
-                .map(|p| filename[p..].to_lowercase())
-                .unwrap_or_default()
-        };
-        if is_compressible_file_type(&ext, &mime_type) {
+        let ext = filename
+            .rfind('.')
+            .map(|p| &filename[p..])
+            .unwrap_or_default();
+        if is_compressible_file_type(ext, &mime_type) {
             if let Some(compressed) = try_gzip_data(&body_data) {
                 if compressed.len() * 10 < body_data.len() * 9 {
                     (compressed, true)
@@ -4728,6 +4725,15 @@ fn extract_jwt(headers: &HeaderMap, uri: &axum::http::Uri) -> Option<String> {
 /// Check if a file type should be compressed based on extension and MIME type.
 /// Returns true only when we are sure the type is compressible.
 fn is_compressible_file_type(ext: &str, mtype: &str) -> bool {
+    let ext = ext.to_ascii_lowercase();
+    let ext = ext.as_str();
+    let mtype = mtype
+        .split(';')
+        .next()
+        .unwrap_or_default()
+        .trim()
+        .to_ascii_lowercase();
+    let mtype = mtype.as_str();
     // text/*
     if mtype.starts_with("text/") {
         return true;
@@ -4760,6 +4766,9 @@ fn is_compressible_file_type(ext: &str, mtype: &str) -> bool {
         if mtype.ends_with("script") {
             return true;
         }
+        if mtype.ends_with("json") || mtype.ends_with("json-seq") {
+            return true;
+        }
         if mtype.ends_with("vnd.rar") {
             return false;
         }
@@ -5245,6 +5254,18 @@ mod tests {
         assert!(is_compressible_file_type("", "application/javascript"));
         assert!(!is_compressible_file_type("", "application/zstd"));
         assert!(!is_compressible_file_type("", "application/vnd.rar"));
+        assert!(is_compressible_file_type("", "application/json"));
+        assert!(is_compressible_file_type(
+            "",
+            "application/json; charset=utf-8"
+        ));
+        assert!(is_compressible_file_type("", "Application/JSON"));
+        assert!(is_compressible_file_type("", "application/ld+json"));
+        assert!(is_compressible_file_type("", "application/x-ndjson"));
+        assert!(is_compressible_file_type("", "application/json-seq"));
+        assert!(is_compressible_file_type("", "text/plain; charset=utf-8"));
+        assert!(is_compressible_file_type(".JSON", ""));
+        assert!(!is_compressible_file_type(".gz", "application/json"));
 
         // Audio
         assert!(is_compressible_file_type(".wav", "audio/wav"));

weed/operation/needle_parse_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"math/rand"
 	"net/http"
 	"testing"
 
@@ -118,6 +119,33 @@ func TestCreateNeedleFromRequest(t *testing.T) {
 
 }
 
+func TestUploadKeepsMarginalGzipUncompressed(t *testing.T) {
+	data := make([]byte, 4300)
+	rand.New(rand.NewSource(1)).Read(data[:4000])
+	gzipped, _ := util.GzipData(data)
+	if len(gzipped) >= len(data) || len(gzipped)*10 < len(data)*9 {
+		t.Fatalf("gzip size %d of %d is not a marginal saving", len(gzipped), len(data))
+	}
+
+	mockClient := &MockClient{}
+	var stored *needle.Needle
+	mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) {
+		assert.Equal(t, nil, err, "upload: %v", err)
+		stored = n
+	}
+	uploadOption := &UploadOption{
+		UploadUrl: "http://localhost:8080/389,0f084d17353afda0",
+		Filename:  "doc.pdf",
+		MimeType:  "application/pdf",
+	}
+	newUploader(mockClient).Upload(context.Background(), bytes.NewReader(data), uploadOption)
+	if stored == nil {
+		t.Fatal("no needle reached the volume server")
+	}
+	assert.False(t, stored.IsCompressed(), "gzip saved less than 10%, data should stay uncompressed")
+	assert.Equal(t, data, stored.Data)
+}
+
 var textContent = `Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
 met:

weed/operation/upload_content.go

@@ -390,7 +390,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
 				option.MimeType = ""
 			}
 		}
-		if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(option.Filename), option.MimeType); iAmSure && shouldBeCompressed {
+		if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Ext(option.Filename), option.MimeType); iAmSure && shouldBeCompressed {
 			shouldGzipNow = true
 		} else if !iAmSure && option.MimeType == "" && len(data) > 16*1024 {
 			var compressed []byte
@@ -412,7 +412,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
 		compressed, compressErr := util.GzipData(data)
 		// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
 		if compressErr == nil {
-			if len(compressed) < len(data) {
+			if len(compressed)*10 < len(data)*9 {
 				data = compressed
 				contentIsGzipped = true
 			}

weed/storage/needle/needle_parse_upload.go

@@ -96,7 +96,7 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p
 		}
 	} else if r.FormValue("type") != "replicate" {
 		// replica writes must keep the source needle's compression state, not re-derive it
-		ext := filepath.Base(pu.FileName)
+		ext := filepath.Ext(pu.FileName)
 		mimeType := pu.MimeType
 		if mimeType == "" {
 			mimeType = http.DetectContentType(pu.Data)

weed/util/compression.go

@@ -99,6 +99,12 @@ func IsZstdContent(data []byte) bool {
 /*
 * Default not to compressed since compression can be done on client side.
  */func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool) {
+	ext = strings.ToLower(ext)
+	// Match on the media type alone: "application/json; charset=utf-8" is JSON.
+	if i := strings.IndexByte(mtype, ';'); i >= 0 {
+		mtype = strings.TrimSpace(mtype[:i])
+	}
+	mtype = strings.ToLower(mtype)
 
 	// text
 	if strings.HasPrefix(mtype, "text/") {
@@ -137,6 +143,10 @@ func IsZstdContent(data []byte) bool {
 		if strings.HasSuffix(mtype, "script") {
 			return true, true
 		}
+		// application/json, application/ld+json, application/x-ndjson, application/json-seq, ...
+		if strings.HasSuffix(mtype, "json") || strings.HasSuffix(mtype, "json-seq") {
+			return true, true
+		}
 		if strings.HasSuffix(mtype, "vnd.rar") {
 			return false, true
 		}

weed/util/compression_test.go

@@ -0,0 +1,31 @@
+package util
+
+import "testing"
+
+func TestIsCompressableFileType(t *testing.T) {
+	tests := []struct {
+		ext, mtype       string
+		compress, isSure bool
+	}{
+		{".json", "", true, true},
+		{".JSON", "", true, true},
+		{"", "application/json", true, true},
+		{"", "application/json; charset=utf-8", true, true},
+		{"", "application/ld+json", true, true},
+		{"", "application/x-ndjson", true, true},
+		{"", "application/json-seq", true, true},
+		{"", "Application/JSON", true, true},
+		{"", "text/plain; charset=utf-8", true, true},
+		{"", "application/xml", true, true},
+		{".gz", "application/json", false, true},
+		{"", "application/gzip", false, false},
+		{"", "image/png", false, true},
+		{"", "", false, false},
+	}
+	for _, tt := range tests {
+		compress, isSure := IsCompressableFileType(tt.ext, tt.mtype)
+		if compress != tt.compress || isSure != tt.isSure {
+			t.Errorf("IsCompressableFileType(%q, %q) = (%v, %v), want (%v, %v)", tt.ext, tt.mtype, compress, isSure, tt.compress, tt.isSure)
+		}
+	}
+}