mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-19 12:34:29 +02:00
filer: shrink zip spool threshold in tests to fix 32-bit OOM (#11796)
TestSpoolZipReaderTempFile and TestSpoolZipReaderLimit each allocate ~33-70 MiB to cross the 32 MiB in-memory spool threshold. On linux/386 the weed/filer test binary already runs near the 4 GiB address-space limit, so these allocations reliably crash it with "out of memory" mid-suite. Make zipSpoolMemoryLimit a var and drop it to 1 KiB in those tests, which exercises the same memory-vs-temp-file spool paths with KiB-sized buffers.
This commit is contained in:
1 parent
1155e65fc5
commit
ac996c0ad1
2 files changed
+21
-3
No files matched your search
@@ -22,9 +22,15 @@ const (
|
||||
// the TUS default max upload size.
|
||||
ZipExtractMaxArchiveBytes = int64(5) << 30 // 5 GiB
|
||||
|
||||
zipSpoolMemoryLimit = 32 << 20 // 32 MiB; larger archives spool to a temp file
|
||||
)
|
||||
|
||||
// zipSpoolMemoryLimit caps how much of one archive is buffered in memory
|
||||
// before the spool falls through to a temp file. It is a var rather than a
|
||||
// const so tests can shrink it: exercising the temp-file path with the real
|
||||
// 32 MiB threshold allocates hundreds of MiB, which tips the 32-bit test
|
||||
// binary over its 4 GiB address space.
|
||||
var zipSpoolMemoryLimit int64 = 32 << 20 // 32 MiB
|
||||
|
||||
// ErrZipTooLarge marks a spool failure caused by the archive exceeding the
|
||||
// caller's size limit, so handlers can answer 4xx instead of 5xx.
|
||||
var ErrZipTooLarge = errors.New("zip archive too large")
|
||||
|
||||
@@ -46,6 +46,16 @@ func TestSanitizeZipMemberName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// smallZipSpoolLimit drops the in-memory spool threshold to 1 KiB for the
|
||||
// duration of a test. Crossing the real 32 MiB threshold needs buffers that
|
||||
// are too large for the 32-bit test binary's address space.
|
||||
func smallZipSpoolLimit(t *testing.T) {
|
||||
t.Helper()
|
||||
old := zipSpoolMemoryLimit
|
||||
zipSpoolMemoryLimit = 1024
|
||||
t.Cleanup(func() { zipSpoolMemoryLimit = old })
|
||||
}
|
||||
|
||||
func TestSpoolZipReaderMemory(t *testing.T) {
|
||||
data := []byte(strings.Repeat("x", 1024))
|
||||
ra, n, cleanup, err := SpoolZipReader(bytes.NewReader(data), 0)
|
||||
@@ -66,7 +76,8 @@ func TestSpoolZipReaderMemory(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSpoolZipReaderTempFile(t *testing.T) {
|
||||
data := bytes.Repeat([]byte("y"), zipSpoolMemoryLimit+1)
|
||||
smallZipSpoolLimit(t)
|
||||
data := bytes.Repeat([]byte("y"), int(zipSpoolMemoryLimit)+1)
|
||||
ra, n, cleanup, err := SpoolZipReader(bytes.NewReader(data), 0)
|
||||
if err != nil {
|
||||
t.Fatalf("spool: %v", err)
|
||||
@@ -87,11 +98,12 @@ func TestSpoolZipReaderTempFile(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSpoolZipReaderLimit(t *testing.T) {
|
||||
smallZipSpoolLimit(t)
|
||||
data := []byte(strings.Repeat("x", 2048))
|
||||
if _, _, _, err := SpoolZipReader(bytes.NewReader(data), 1024); err == nil {
|
||||
t.Fatal("expected limit error for in-memory archive")
|
||||
}
|
||||
if _, _, cleanup, err := SpoolZipReader(bytes.NewReader(bytes.Repeat([]byte("y"), zipSpoolMemoryLimit+1024)), zipSpoolMemoryLimit+512); err == nil {
|
||||
if _, _, cleanup, err := SpoolZipReader(bytes.NewReader(bytes.Repeat([]byte("y"), int(zipSpoolMemoryLimit)+1024)), zipSpoolMemoryLimit+512); err == nil {
|
||||
t.Fatal("expected limit error for spooled archive")
|
||||
} else {
|
||||
cleanup()
|
||||
|
||||
Reference in new issue
Block a user