mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-31 09:20:50 +01:00
Chris Lu
6b379badbb
* filer: shut down test filers' meta-log buffers, fixing the 32-bit OOM Each log_buffer.NewLogBuffer preallocates five 8 MiB windows and spawns loopFlush/loopInterval goroutines that keep them reachable. The weed/filer tests built ~70 filers and aggregators that almost never called ShutdownLogBuffer, pinning ~2.8 GB plus live churn, and the linux/386 job ran out of its ~4 GiB address space inside newSealedBuffers. - MetaAggregator.Shutdown now retires MetaLogBuffer too: the "aggr" buffer's loops previously outlived the aggregator. - newTestFiler and newTestFilerWithStubStore register t.Cleanup shutdowns; the one-off filer/aggregator literals defer theirs. * filer: wait for the aggregated meta-log buffer to stop in Shutdown ShutdownLogBuffer only signals the stop; loopFlush and its pooled windows exit asynchronously. Match Filer.Shutdown's local-buffer sequence so the aggregator returns with the buffer's memory already released. * filer: cancel test filers' flush context before buffer shutdown The stub-store helpers build real NewFiler instances, so the queued shutdown flush retries appendToFile every ~737ms under flushCtx, which nothing ever cancelled — the buffer and its pool stayed alive past each test. Cancelling first lets the flush drop the window and WaitForShutdown confirms loopFlush exited. * filer: wait for test log buffers to drain after shutdown The stub-flush buffers cannot hang, but without WaitForShutdown their loopFlush goroutines release the pooled windows asynchronously — the memory this PR means to free is still counted as live while the next test allocates.
96 lines
2.5 KiB
Go
96 lines
2.5 KiB
Go
package filer
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
|
|
)
|
|
|
|
func TestNotifyUpdateEventRecordsRequestMetadataEvent(t *testing.T) {
|
|
f := &Filer{
|
|
Signature: 42,
|
|
LocalMetaLogBuffer: log_buffer.NewLogBuffer(
|
|
"test",
|
|
time.Hour,
|
|
func(*log_buffer.LogBuffer, time.Time, time.Time, []byte, int64, int64) {},
|
|
nil,
|
|
nil,
|
|
),
|
|
}
|
|
defer func() {
|
|
f.LocalMetaLogBuffer.ShutdownLogBuffer()
|
|
f.LocalMetaLogBuffer.WaitForShutdown()
|
|
}()
|
|
|
|
ctx, sink := WithMetadataEventSink(context.Background())
|
|
f.NotifyUpdateEvent(ctx, &Entry{FullPath: util.FullPath("/dir/file.txt")}, nil, true, false, []int32{7})
|
|
|
|
event := sink.Last()
|
|
if event == nil {
|
|
t.Fatal("expected metadata event to be recorded")
|
|
}
|
|
if event.Directory != "/dir" {
|
|
t.Fatalf("directory = %q, want /dir", event.Directory)
|
|
}
|
|
if event.EventNotification.OldEntry == nil || event.EventNotification.OldEntry.Name != "file.txt" {
|
|
t.Fatalf("old entry = %+v, want file.txt", event.EventNotification.OldEntry)
|
|
}
|
|
if got := event.EventNotification.Signatures; len(got) != 2 || got[0] != 7 || got[1] != 42 {
|
|
t.Fatalf("signatures = %v, want [7 42]", got)
|
|
}
|
|
if event.TsNs == 0 {
|
|
t.Fatal("expected event timestamp to be set")
|
|
}
|
|
}
|
|
|
|
func TestNotifyUpdateEventReloadsLocalFilerConfiguration(t *testing.T) {
|
|
f := &Filer{
|
|
Signature: 42,
|
|
FilerConf: NewFilerConf(),
|
|
LocalMetaLogBuffer: log_buffer.NewLogBuffer(
|
|
"test",
|
|
time.Hour,
|
|
func(*log_buffer.LogBuffer, time.Time, time.Time, []byte, int64, int64) {},
|
|
nil,
|
|
nil,
|
|
),
|
|
}
|
|
defer func() {
|
|
f.LocalMetaLogBuffer.ShutdownLogBuffer()
|
|
f.LocalMetaLogBuffer.WaitForShutdown()
|
|
}()
|
|
|
|
updatedConf := NewFilerConf()
|
|
if err := updatedConf.SetLocationConf(&filer_pb.FilerConf_PathConf{
|
|
LocationPrefix: "/data/",
|
|
Collection: "hot",
|
|
}); err != nil {
|
|
t.Fatalf("set location conf: %v", err)
|
|
}
|
|
var content bytes.Buffer
|
|
if err := updatedConf.ToText(&content); err != nil {
|
|
t.Fatalf("serialize filer conf: %v", err)
|
|
}
|
|
|
|
f.NotifyUpdateEvent(context.Background(), nil, &Entry{
|
|
FullPath: util.NewFullPath(DirectoryEtcSeaweedFS, FilerConfName),
|
|
Attr: Attr{
|
|
FileSize: uint64(content.Len()),
|
|
},
|
|
Content: content.Bytes(),
|
|
}, false, false, nil)
|
|
|
|
locConf, found := f.FilerConf.GetLocationConf("/data/")
|
|
if !found {
|
|
t.Fatal("expected local filer configuration to reload /data/ rule")
|
|
}
|
|
if locConf.Collection != "hot" {
|
|
t.Fatalf("collection = %q, want hot", locConf.Collection)
|
|
}
|
|
}
|