mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-07 12:37:41 +01:00
1176f975b3
* filer: stop unbounded metadata-log growth on append-only files Two independent leaks make a filer metadata log grow far faster than the data it describes. On a cluster whose live logs weigh a few hundred megabytes, they produced tens of gigabytes of metadata per day. 1. Metadata events embed the whole old and new entry, chunk lists included. An append-only file — a log or a journal written in small pieces — therefore re-serializes every chunk it ever had on every append, so the cost is quadratic in the number of chunks. A 2,700-chunk file emits a couple of megabytes per append. Cap the carried chunk list at 256 entries and set metadata_event_chunks_truncated so subscribers that need the content read the entry from the filer instead of trusting the event. Subscribers that only track names, attributes, or chunk deltas keep working unchanged. 2. appendToFile uploads the chunk before committing the filer entry. When the commit fails — store error, cancelled context, filer shutting down — the chunk is stranded: no entry references it, so no later delete can mark it, and neither vacuum nor fsck can see it because both work from the entry index. The volume server keeps counting those bytes as live data forever. Release the chunk when the commit fails. * filer: keep chunk lists in metadata events Mount meta caches, peer filers replaying into their own store, and the filer.sync, filer.backup and filer.meta.backup sinks all store the event's entry as the file. Dropping the chunk list past a count left those copies of a large file without content. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: release a metadata log chunk the lookup or commit leaves unreferenced The flush retries a failed append with a new upload, so a store outage left one stranded chunk per retry. A failed lookup strands the chunk too. Use DeleteUncommittedChunks, as the other write paths do: the chunk was never part of the file, so path chunk-deletion rules do not apply. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: keep a metadata log chunk the failed create stored CreateEntry inserts a new entry before its parent folders, so a parent failure returns an error with the entry already referencing the chunk. Release it only when the store shows no entry, or an entry without it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: count a metadata log piece the failed create stored as appended The flush retry would append the same bytes after it, and a piece cut inside a record then breaks replay of that segment. The retry takes the update path, which does not create parent folders either, so returning the error only duplicated the piece. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Martin Morez <martin@morez.org> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
82 lines
3.0 KiB
Go
82 lines
3.0 KiB
Go
package filer
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
const appendTestLog = util.FullPath("/topics/.system/log/2026-11-05/10-00.00000001")
|
|
|
|
func newAppendTestFiler(store *stubFilerStore) *Filer {
|
|
f := NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
|
|
f.Store = NewFilerStoreWrapper(store)
|
|
return f
|
|
}
|
|
|
|
func queuedForDeletion(f *Filer, fid string) bool {
|
|
f.deletionLedgerLock.Lock()
|
|
defer f.deletionLedgerLock.Unlock()
|
|
_, queued := f.pendingDeletions[fid]
|
|
return queued
|
|
}
|
|
|
|
func TestCommitAppendedChunkReleasesChunkWhenCommitFails(t *testing.T) {
|
|
store := newStubFilerStore()
|
|
store.insertErr = errors.New("store unavailable")
|
|
f := newAppendTestFiler(store)
|
|
|
|
err := f.commitAppendedChunk(context.Background(), appendTestLog, &filer_pb.FileChunk{FileId: "7,0abc", Size: 5})
|
|
|
|
require.Error(t, err)
|
|
assert.True(t, queuedForDeletion(f, "7,0abc"), "a chunk no entry references must be released")
|
|
}
|
|
|
|
func TestCommitAppendedChunkReleasesChunkWhenLookupFails(t *testing.T) {
|
|
store := newStubFilerStore()
|
|
store.findErrByPath = map[string]error{string(appendTestLog): errors.New("store unavailable")}
|
|
f := newAppendTestFiler(store)
|
|
|
|
err := f.commitAppendedChunk(context.Background(), appendTestLog, &filer_pb.FileChunk{FileId: "7,0abc", Size: 5})
|
|
|
|
require.Error(t, err)
|
|
assert.True(t, queuedForDeletion(f, "7,0abc"), "a chunk no entry references must be released")
|
|
}
|
|
|
|
func TestCommitAppendedChunkKeepsChunkTheFailedCreateStored(t *testing.T) {
|
|
store := newStubFilerStore()
|
|
parent, _ := appendTestLog.DirAndName()
|
|
store.findErrByPath = map[string]error{parent: errors.New("store unavailable")}
|
|
f := newAppendTestFiler(store)
|
|
|
|
err := f.commitAppendedChunk(context.Background(), appendTestLog, &filer_pb.FileChunk{FileId: "7,0abc", Size: 5})
|
|
|
|
require.NoError(t, err, "a stored piece must not be appended again by the flush retry")
|
|
stored, findErr := store.FindEntry(context.Background(), appendTestLog)
|
|
require.NoError(t, findErr, "the entry is stored before its parent folders are created")
|
|
require.Len(t, stored.Chunks, 1)
|
|
assert.False(t, queuedForDeletion(f, "7,0abc"), "a chunk the stored entry references must stay")
|
|
}
|
|
|
|
func TestCommitAppendedChunkAppendsAfterExistingChunks(t *testing.T) {
|
|
store := newStubFilerStore()
|
|
f := newAppendTestFiler(store)
|
|
ctx := context.Background()
|
|
|
|
require.NoError(t, f.commitAppendedChunk(ctx, appendTestLog, &filer_pb.FileChunk{FileId: "7,0abc", Size: 5}))
|
|
require.NoError(t, f.commitAppendedChunk(ctx, appendTestLog, &filer_pb.FileChunk{FileId: "7,0abd", Size: 3}))
|
|
|
|
stored, err := store.FindEntry(ctx, appendTestLog)
|
|
require.NoError(t, err)
|
|
require.Len(t, stored.Chunks, 2)
|
|
assert.Equal(t, int64(5), stored.Chunks[1].Offset)
|
|
assert.False(t, queuedForDeletion(f, "7,0abc"))
|
|
assert.False(t, queuedForDeletion(f, "7,0abd"))
|
|
}
|