From ac996c0ad1824f23d30be2b07e305de7dfb82927 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 17 Oct 2026 11:43:55 -0700 Subject: [PATCH] filer: shrink zip spool threshold in tests to fix 32-bit OOM (#11796) TestSpoolZipReaderTempFile and TestSpoolZipReaderLimit each allocate ~33-70 MiB to cross the 32 MiB in-memory spool threshold. On linux/386 the weed/filer test binary already runs near the 4 GiB address-space limit, so these allocations reliably crash it with "out of memory" mid-suite. Make zipSpoolMemoryLimit a var and drop it to 1 KiB in those tests, which exercises the same memory-vs-temp-file spool paths with KiB-sized buffers. --- weed/filer/zip_extract.go | 8 +++++++- weed/filer/zip_extract_test.go | 16 ++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/weed/filer/zip_extract.go b/weed/filer/zip_extract.go index 99ed3a624..28096884b 100644 --- a/weed/filer/zip_extract.go +++ b/weed/filer/zip_extract.go @@ -22,9 +22,15 @@ const ( // the TUS default max upload size. ZipExtractMaxArchiveBytes = int64(5) << 30 // 5 GiB - zipSpoolMemoryLimit = 32 << 20 // 32 MiB; larger archives spool to a temp file ) +// zipSpoolMemoryLimit caps how much of one archive is buffered in memory +// before the spool falls through to a temp file. It is a var rather than a +// const so tests can shrink it: exercising the temp-file path with the real +// 32 MiB threshold allocates hundreds of MiB, which tips the 32-bit test +// binary over its 4 GiB address space. +var zipSpoolMemoryLimit int64 = 32 << 20 // 32 MiB + // ErrZipTooLarge marks a spool failure caused by the archive exceeding the // caller's size limit, so handlers can answer 4xx instead of 5xx. var ErrZipTooLarge = errors.New("zip archive too large") diff --git a/weed/filer/zip_extract_test.go b/weed/filer/zip_extract_test.go index e91b3a635..89fd273cc 100644 --- a/weed/filer/zip_extract_test.go +++ b/weed/filer/zip_extract_test.go @@ -46,6 +46,16 @@ func TestSanitizeZipMemberName(t *testing.T) { } } +// smallZipSpoolLimit drops the in-memory spool threshold to 1 KiB for the +// duration of a test. Crossing the real 32 MiB threshold needs buffers that +// are too large for the 32-bit test binary's address space. +func smallZipSpoolLimit(t *testing.T) { + t.Helper() + old := zipSpoolMemoryLimit + zipSpoolMemoryLimit = 1024 + t.Cleanup(func() { zipSpoolMemoryLimit = old }) +} + func TestSpoolZipReaderMemory(t *testing.T) { data := []byte(strings.Repeat("x", 1024)) ra, n, cleanup, err := SpoolZipReader(bytes.NewReader(data), 0) @@ -66,7 +76,8 @@ func TestSpoolZipReaderMemory(t *testing.T) { } func TestSpoolZipReaderTempFile(t *testing.T) { - data := bytes.Repeat([]byte("y"), zipSpoolMemoryLimit+1) + smallZipSpoolLimit(t) + data := bytes.Repeat([]byte("y"), int(zipSpoolMemoryLimit)+1) ra, n, cleanup, err := SpoolZipReader(bytes.NewReader(data), 0) if err != nil { t.Fatalf("spool: %v", err) @@ -87,11 +98,12 @@ func TestSpoolZipReaderTempFile(t *testing.T) { } func TestSpoolZipReaderLimit(t *testing.T) { + smallZipSpoolLimit(t) data := []byte(strings.Repeat("x", 2048)) if _, _, _, err := SpoolZipReader(bytes.NewReader(data), 1024); err == nil { t.Fatal("expected limit error for in-memory archive") } - if _, _, cleanup, err := SpoolZipReader(bytes.NewReader(bytes.Repeat([]byte("y"), zipSpoolMemoryLimit+1024)), zipSpoolMemoryLimit+512); err == nil { + if _, _, cleanup, err := SpoolZipReader(bytes.NewReader(bytes.Repeat([]byte("y"), int(zipSpoolMemoryLimit)+1024)), zipSpoolMemoryLimit+512); err == nil { t.Fatal("expected limit error for spooled archive") } else { cleanup()