Files
seaweedfs/weed/filer/filer_prefix_object_test.go
T
Chris Lu 6b379badbb filer: shut down test filers' meta-log buffers, fixing the 32-bit OOM (#12104)
* filer: shut down test filers' meta-log buffers, fixing the 32-bit OOM

Each log_buffer.NewLogBuffer preallocates five 8 MiB windows and spawns
loopFlush/loopInterval goroutines that keep them reachable. The
weed/filer tests built ~70 filers and aggregators that almost never
called ShutdownLogBuffer, pinning ~2.8 GB plus live churn, and the
linux/386 job ran out of its ~4 GiB address space inside
newSealedBuffers.

- MetaAggregator.Shutdown now retires MetaLogBuffer too: the "aggr"
  buffer's loops previously outlived the aggregator.
- newTestFiler and newTestFilerWithStubStore register t.Cleanup
  shutdowns; the one-off filer/aggregator literals defer theirs.

* filer: wait for the aggregated meta-log buffer to stop in Shutdown

ShutdownLogBuffer only signals the stop; loopFlush and its pooled windows
exit asynchronously. Match Filer.Shutdown's local-buffer sequence so the
aggregator returns with the buffer's memory already released.

* filer: cancel test filers' flush context before buffer shutdown

The stub-store helpers build real NewFiler instances, so the queued
shutdown flush retries appendToFile every ~737ms under flushCtx, which
nothing ever cancelled — the buffer and its pool stayed alive past each
test. Cancelling first lets the flush drop the window and WaitForShutdown
confirms loopFlush exited.

* filer: wait for test log buffers to drain after shutdown

The stub-flush buffers cannot hang, but without WaitForShutdown their
loopFlush goroutines release the pooled windows asynchronously — the
memory this PR means to free is still counted as live while the next
test allocates.
2026-10-26 13:24:48 -07:00

127 lines
4.1 KiB
Go

package filer
import (
"context"
"os"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestPromoteToPrefixObject covers a key written before the keys nested under it:
// the file becomes the directory they live in, and has to stay an object of its own.
func TestPromoteToPrefixObject(t *testing.T) {
f, store := newTestFilerWithStubStore(t)
ctx := context.Background()
object := &Entry{
FullPath: util.FullPath("/buckets/bkt/a/foo"),
Attr: Attr{
Mode: 0o644,
Mime: "text/plain",
TtlSec: 3600,
},
Chunks: []*filer_pb.FileChunk{{FileId: "1,01", Size: 4}},
Extended: map[string][]byte{s3_constants.SeaweedFSExpiresS3: []byte("true")},
}
require.NoError(t, f.CreateEntry(ctx, object, nil, false, false, nil, false, f.MaxFilenameLength))
nested := &Entry{FullPath: util.FullPath("/buckets/bkt/a/foo/bar"), Attr: Attr{Mode: 0o644}}
require.NoError(t, f.CreateEntry(ctx, nested, nil, false, false, nil, false, f.MaxFilenameLength))
promoted, err := store.FindEntry(ctx, object.FullPath)
require.NoError(t, err)
require.NotNil(t, promoted)
assert.True(t, promoted.IsDirectory(), "the nested key needs a directory here")
assert.Equal(t, object.Chunks, promoted.Chunks, "the object's data stays on it")
assert.Contains(t, promoted.Extended, s3_constants.SeaweedFSPrefixObject)
// Expiring the entry deletes the directory row on its own and strands the keys
// under it, so the promotion gives up the lazy TTL.
assert.Zero(t, promoted.Attr.TtlSec)
}
// TestExpiredDirectoryIsNotDeletedOnRead pins the other half: a TTL an older build
// left on a promoted directory must not take the keys under it with it.
func TestExpiredDirectoryIsNotDeletedOnRead(t *testing.T) {
f, store := newTestFilerWithStubStore(t)
ctx := context.Background()
dirPath := util.FullPath("/buckets/bkt/a/foo")
expired := time.Now().Add(-2 * time.Hour)
require.NoError(t, store.InsertEntry(ctx, &Entry{
FullPath: dirPath,
Attr: Attr{
Mode: os.ModeDir | 0o755,
Crtime: expired,
Mtime: expired,
TtlSec: 60,
},
Extended: map[string][]byte{s3_constants.SeaweedFSExpiresS3: []byte("true")},
}))
nested := &Entry{FullPath: dirPath + "/bar", Attr: Attr{Mode: 0o644}}
require.NoError(t, store.InsertEntry(ctx, nested))
found, err := f.FindEntry(ctx, dirPath)
require.NoError(t, err)
require.NotNil(t, found, "deleting it here would leave the nested key unreachable")
stillThere, err := store.FindEntry(ctx, nested.FullPath)
require.NoError(t, err)
require.NotNil(t, stillThere)
}
// TestExpiredFileIsDeletedOnRead pins the native cache-TTL path: regular
// entries become invisible and their metadata row is removed without a
// cache-specific transaction or reverse Volume index.
func TestExpiredFileIsDeletedOnRead(t *testing.T) {
f, store := newTestFilerWithStubStore(t)
ctx := context.Background()
filePath := util.FullPath("/buckets/lmcache/expired")
expired := time.Now().Add(-2 * time.Hour)
require.NoError(t, store.InsertEntry(ctx, &Entry{
FullPath: filePath,
Attr: Attr{
Mode: 0o644,
Crtime: expired,
Mtime: expired,
TtlSec: 60,
},
Chunks: []*filer_pb.FileChunk{{FileId: "7,01", Size: 4}},
}))
found, err := f.FindEntry(ctx, filePath)
require.ErrorIs(t, err, filer_pb.ErrNotFound)
require.Nil(t, found)
_, err = store.FindEntry(ctx, filePath)
require.ErrorIs(t, err, filer_pb.ErrNotFound,
"native TTL lookup should remove the expired metadata row")
}
func TestListDirectoryEntriesContextCanceled(t *testing.T) {
f, store := newTestFilerWithStubStore(t)
ctx := context.Background()
require.NoError(t, store.InsertEntry(ctx, &Entry{
FullPath: util.FullPath("/dir/file1.txt"),
Attr: Attr{Mode: 0o644},
}))
canceledCtx, cancel := context.WithCancel(ctx)
cancel()
_, _, err := f.doListDirectoryEntries(canceledCtx, util.FullPath("/dir"), "", false, 10, "", func(entry *Entry) (bool, error) {
return true, nil
})
require.Error(t, err)
assert.ErrorContains(t, err, "context canceled")
}