filer: shrink zip spool threshold in tests to fix 32-bit OOM (#11796)

TestSpoolZipReaderTempFile and TestSpoolZipReaderLimit each allocate
~33-70 MiB to cross the 32 MiB in-memory spool threshold. On linux/386 the
weed/filer test binary already runs near the 4 GiB address-space limit, so
these allocations reliably crash it with "out of memory" mid-suite.

Make zipSpoolMemoryLimit a var and drop it to 1 KiB in those tests, which
exercises the same memory-vs-temp-file spool paths with KiB-sized buffers.
This commit is contained in:
Chris Lu authored and GitHub committed 2026-10-17 11:43:55 -07:00
1 parent 1155e65fc5
commit ac996c0ad1
2 files changed
+21 -3

No files matched your search

+7 -1
View File
@@ -22,9 +22,15 @@ const (
// the TUS default max upload size.
ZipExtractMaxArchiveBytes = int64(5) << 30 // 5 GiB
zipSpoolMemoryLimit = 32 << 20 // 32 MiB; larger archives spool to a temp file
)
// zipSpoolMemoryLimit caps how much of one archive is buffered in memory
// before the spool falls through to a temp file. It is a var rather than a
// const so tests can shrink it: exercising the temp-file path with the real
// 32 MiB threshold allocates hundreds of MiB, which tips the 32-bit test
// binary over its 4 GiB address space.
var zipSpoolMemoryLimit int64 = 32 << 20 // 32 MiB
// ErrZipTooLarge marks a spool failure caused by the archive exceeding the
// caller's size limit, so handlers can answer 4xx instead of 5xx.
var ErrZipTooLarge = errors.New("zip archive too large")
+14 -2
View File
@@ -46,6 +46,16 @@ func TestSanitizeZipMemberName(t *testing.T) {
}
}
// smallZipSpoolLimit drops the in-memory spool threshold to 1 KiB for the
// duration of a test. Crossing the real 32 MiB threshold needs buffers that
// are too large for the 32-bit test binary's address space.
func smallZipSpoolLimit(t *testing.T) {
t.Helper()
old := zipSpoolMemoryLimit
zipSpoolMemoryLimit = 1024
t.Cleanup(func() { zipSpoolMemoryLimit = old })
}
func TestSpoolZipReaderMemory(t *testing.T) {
data := []byte(strings.Repeat("x", 1024))
ra, n, cleanup, err := SpoolZipReader(bytes.NewReader(data), 0)
@@ -66,7 +76,8 @@ func TestSpoolZipReaderMemory(t *testing.T) {
}
func TestSpoolZipReaderTempFile(t *testing.T) {
data := bytes.Repeat([]byte("y"), zipSpoolMemoryLimit+1)
smallZipSpoolLimit(t)
data := bytes.Repeat([]byte("y"), int(zipSpoolMemoryLimit)+1)
ra, n, cleanup, err := SpoolZipReader(bytes.NewReader(data), 0)
if err != nil {
t.Fatalf("spool: %v", err)
@@ -87,11 +98,12 @@ func TestSpoolZipReaderTempFile(t *testing.T) {
}
func TestSpoolZipReaderLimit(t *testing.T) {
smallZipSpoolLimit(t)
data := []byte(strings.Repeat("x", 2048))
if _, _, _, err := SpoolZipReader(bytes.NewReader(data), 1024); err == nil {
t.Fatal("expected limit error for in-memory archive")
}
if _, _, cleanup, err := SpoolZipReader(bytes.NewReader(bytes.Repeat([]byte("y"), zipSpoolMemoryLimit+1024)), zipSpoolMemoryLimit+512); err == nil {
if _, _, cleanup, err := SpoolZipReader(bytes.NewReader(bytes.Repeat([]byte("y"), int(zipSpoolMemoryLimit)+1024)), zipSpoolMemoryLimit+512); err == nil {
t.Fatal("expected limit error for spooled archive")
} else {
cleanup()