From d650abe0e76acda59728f3263c314c65d8ae31e5 Mon Sep 17 00:00:00 2001 From: Pat White Date: Mon, 2 Nov 2026 10:15:01 -0800 Subject: [PATCH] compression: compress JSON uploads, and match file types by extension (#12339) * compression: compress JSON uploads, and match file types by extension IsCompressableFileType did not recognise JSON media types, so S3 uploads of application/json (which carry no file name) were never compressed. Its callers also passed filepath.Base of the file name where it expects the extension, so the .json/.js/.html/... extension rules never matched a real file name. Both callers now pass filepath.Ext; the function lower-cases the extension and media type, ignores media-type parameters ("application/json; charset=utf-8"), and treats */json and *+json types as compressible. * compression: treat application/json-seq as JSON Co-Authored-By: Claude Opus 5.5 * volume(rust): compress JSON uploads and match media types without parameters Co-Authored-By: Claude Opus 5.5 * operation: keep uploader gzip only when it saves at least 10% Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Pat White <5015863+patwhite@users.noreply.github.com> Co-authored-by: Chris Lu Co-authored-by: Claude Opus 5.5 --- seaweed-volume/src/server/handlers.rs | 37 +++++++++++++++++----- weed/operation/needle_parse_test.go | 28 ++++++++++++++++ weed/operation/upload_content.go | 4 +-- weed/storage/needle/needle_parse_upload.go | 2 +- weed/util/compression.go | 10 ++++++ weed/util/compression_test.go | 31 ++++++++++++++++++ 6 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 weed/util/compression_test.go diff --git a/seaweed-volume/src/server/handlers.rs b/seaweed-volume/src/server/handlers.rs index d2d02d35b0..91c2d2a7ca 100644 --- a/seaweed-volume/src/server/handlers.rs +++ b/seaweed-volume/src/server/handlers.rs @@ -3101,15 +3101,12 @@ pub async fn post_handler( // Auto-compress compressible file types (matches Go's IsCompressableFileType). // Only compress if not already gzipped and compression saves >10%. - // Go uses filepath.Base(pu.FileName) for extension detection (not the URL path). let (final_data, final_is_gzipped) = if !is_gzipped && !is_chunk_manifest { - let ext = { - let dot_pos = filename.rfind('.'); - dot_pos - .map(|p| filename[p..].to_lowercase()) - .unwrap_or_default() - }; - if is_compressible_file_type(&ext, &mime_type) { + let ext = filename + .rfind('.') + .map(|p| &filename[p..]) + .unwrap_or_default(); + if is_compressible_file_type(ext, &mime_type) { if let Some(compressed) = try_gzip_data(&body_data) { if compressed.len() * 10 < body_data.len() * 9 { (compressed, true) @@ -4728,6 +4725,15 @@ fn extract_jwt(headers: &HeaderMap, uri: &axum::http::Uri) -> Option { /// Check if a file type should be compressed based on extension and MIME type. /// Returns true only when we are sure the type is compressible. fn is_compressible_file_type(ext: &str, mtype: &str) -> bool { + let ext = ext.to_ascii_lowercase(); + let ext = ext.as_str(); + let mtype = mtype + .split(';') + .next() + .unwrap_or_default() + .trim() + .to_ascii_lowercase(); + let mtype = mtype.as_str(); // text/* if mtype.starts_with("text/") { return true; @@ -4760,6 +4766,9 @@ fn is_compressible_file_type(ext: &str, mtype: &str) -> bool { if mtype.ends_with("script") { return true; } + if mtype.ends_with("json") || mtype.ends_with("json-seq") { + return true; + } if mtype.ends_with("vnd.rar") { return false; } @@ -5245,6 +5254,18 @@ mod tests { assert!(is_compressible_file_type("", "application/javascript")); assert!(!is_compressible_file_type("", "application/zstd")); assert!(!is_compressible_file_type("", "application/vnd.rar")); + assert!(is_compressible_file_type("", "application/json")); + assert!(is_compressible_file_type( + "", + "application/json; charset=utf-8" + )); + assert!(is_compressible_file_type("", "Application/JSON")); + assert!(is_compressible_file_type("", "application/ld+json")); + assert!(is_compressible_file_type("", "application/x-ndjson")); + assert!(is_compressible_file_type("", "application/json-seq")); + assert!(is_compressible_file_type("", "text/plain; charset=utf-8")); + assert!(is_compressible_file_type(".JSON", "")); + assert!(!is_compressible_file_type(".gz", "application/json")); // Audio assert!(is_compressible_file_type(".wav", "audio/wav")); diff --git a/weed/operation/needle_parse_test.go b/weed/operation/needle_parse_test.go index 339d4507e2..fb1d530a78 100644 --- a/weed/operation/needle_parse_test.go +++ b/weed/operation/needle_parse_test.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "io" + "math/rand" "net/http" "testing" @@ -118,6 +119,33 @@ func TestCreateNeedleFromRequest(t *testing.T) { } +func TestUploadKeepsMarginalGzipUncompressed(t *testing.T) { + data := make([]byte, 4300) + rand.New(rand.NewSource(1)).Read(data[:4000]) + gzipped, _ := util.GzipData(data) + if len(gzipped) >= len(data) || len(gzipped)*10 < len(data)*9 { + t.Fatalf("gzip size %d of %d is not a marginal saving", len(gzipped), len(data)) + } + + mockClient := &MockClient{} + var stored *needle.Needle + mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) { + assert.Equal(t, nil, err, "upload: %v", err) + stored = n + } + uploadOption := &UploadOption{ + UploadUrl: "http://localhost:8080/389,0f084d17353afda0", + Filename: "doc.pdf", + MimeType: "application/pdf", + } + newUploader(mockClient).Upload(context.Background(), bytes.NewReader(data), uploadOption) + if stored == nil { + t.Fatal("no needle reached the volume server") + } + assert.False(t, stored.IsCompressed(), "gzip saved less than 10%, data should stay uncompressed") + assert.Equal(t, data, stored.Data) +} + var textContent = `Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index a44dc38e8e..4d829e33e7 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -390,7 +390,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option option.MimeType = "" } } - if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(option.Filename), option.MimeType); iAmSure && shouldBeCompressed { + if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Ext(option.Filename), option.MimeType); iAmSure && shouldBeCompressed { shouldGzipNow = true } else if !iAmSure && option.MimeType == "" && len(data) > 16*1024 { var compressed []byte @@ -412,7 +412,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option compressed, compressErr := util.GzipData(data) // fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed)) if compressErr == nil { - if len(compressed) < len(data) { + if len(compressed)*10 < len(data)*9 { data = compressed contentIsGzipped = true } diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go index 0707acef14..3a668de9b6 100644 --- a/weed/storage/needle/needle_parse_upload.go +++ b/weed/storage/needle/needle_parse_upload.go @@ -96,7 +96,7 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p } } else if r.FormValue("type") != "replicate" { // replica writes must keep the source needle's compression state, not re-derive it - ext := filepath.Base(pu.FileName) + ext := filepath.Ext(pu.FileName) mimeType := pu.MimeType if mimeType == "" { mimeType = http.DetectContentType(pu.Data) diff --git a/weed/util/compression.go b/weed/util/compression.go index d62ba90883..aac6b1e012 100644 --- a/weed/util/compression.go +++ b/weed/util/compression.go @@ -99,6 +99,12 @@ func IsZstdContent(data []byte) bool { /* * Default not to compressed since compression can be done on client side. */func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool) { + ext = strings.ToLower(ext) + // Match on the media type alone: "application/json; charset=utf-8" is JSON. + if i := strings.IndexByte(mtype, ';'); i >= 0 { + mtype = strings.TrimSpace(mtype[:i]) + } + mtype = strings.ToLower(mtype) // text if strings.HasPrefix(mtype, "text/") { @@ -137,6 +143,10 @@ func IsZstdContent(data []byte) bool { if strings.HasSuffix(mtype, "script") { return true, true } + // application/json, application/ld+json, application/x-ndjson, application/json-seq, ... + if strings.HasSuffix(mtype, "json") || strings.HasSuffix(mtype, "json-seq") { + return true, true + } if strings.HasSuffix(mtype, "vnd.rar") { return false, true } diff --git a/weed/util/compression_test.go b/weed/util/compression_test.go new file mode 100644 index 0000000000..e9486be3e1 --- /dev/null +++ b/weed/util/compression_test.go @@ -0,0 +1,31 @@ +package util + +import "testing" + +func TestIsCompressableFileType(t *testing.T) { + tests := []struct { + ext, mtype string + compress, isSure bool + }{ + {".json", "", true, true}, + {".JSON", "", true, true}, + {"", "application/json", true, true}, + {"", "application/json; charset=utf-8", true, true}, + {"", "application/ld+json", true, true}, + {"", "application/x-ndjson", true, true}, + {"", "application/json-seq", true, true}, + {"", "Application/JSON", true, true}, + {"", "text/plain; charset=utf-8", true, true}, + {"", "application/xml", true, true}, + {".gz", "application/json", false, true}, + {"", "application/gzip", false, false}, + {"", "image/png", false, true}, + {"", "", false, false}, + } + for _, tt := range tests { + compress, isSure := IsCompressableFileType(tt.ext, tt.mtype) + if compress != tt.compress || isSure != tt.isSure { + t.Errorf("IsCompressableFileType(%q, %q) = (%v, %v), want (%v, %v)", tt.ext, tt.mtype, compress, isSure, tt.compress, tt.isSure) + } + } +}