package filer import ( "bytes" "context" "fmt" "io" "math" "net/http" "net/http/httptest" "strconv" "sync/atomic" "testing" "github.com/stretchr/testify/assert" "google.golang.org/protobuf/proto" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" ) func TestDoMaybeManifestize(t *testing.T) { var manifestTests = []struct { inputs []*filer_pb.FileChunk expected []*filer_pb.FileChunk }{ { inputs: []*filer_pb.FileChunk{ {FileId: "1", IsChunkManifest: false}, {FileId: "2", IsChunkManifest: false}, {FileId: "3", IsChunkManifest: false}, {FileId: "4", IsChunkManifest: false}, }, expected: []*filer_pb.FileChunk{ {FileId: "12", IsChunkManifest: true}, {FileId: "34", IsChunkManifest: true}, }, }, { inputs: []*filer_pb.FileChunk{ {FileId: "1", IsChunkManifest: true}, {FileId: "2", IsChunkManifest: false}, {FileId: "3", IsChunkManifest: false}, {FileId: "4", IsChunkManifest: false}, }, expected: []*filer_pb.FileChunk{ {FileId: "1", IsChunkManifest: true}, {FileId: "23", IsChunkManifest: true}, {FileId: "4", IsChunkManifest: false}, }, }, { inputs: []*filer_pb.FileChunk{ {FileId: "1", IsChunkManifest: false}, {FileId: "2", IsChunkManifest: true}, {FileId: "3", IsChunkManifest: false}, {FileId: "4", IsChunkManifest: false}, }, expected: []*filer_pb.FileChunk{ {FileId: "2", IsChunkManifest: true}, {FileId: "13", IsChunkManifest: true}, {FileId: "4", IsChunkManifest: false}, }, }, { inputs: []*filer_pb.FileChunk{ {FileId: "1", IsChunkManifest: true}, {FileId: "2", IsChunkManifest: true}, {FileId: "3", IsChunkManifest: false}, {FileId: "4", IsChunkManifest: false}, }, expected: []*filer_pb.FileChunk{ {FileId: "1", IsChunkManifest: true}, {FileId: "2", IsChunkManifest: true}, {FileId: "34", IsChunkManifest: true}, }, }, } for i, mtest := range manifestTests { println("test", i) actual, _ := doMaybeManifestize(nil, mtest.inputs, 2, mockMerge) assertEqualChunks(t, mtest.expected, actual) } } func assertEqualChunks(t *testing.T, expected, actual []*filer_pb.FileChunk) { assert.Equal(t, len(expected), len(actual)) for i := 0; i < len(actual); i++ { assertEqualChunk(t, actual[i], expected[i]) } } func assertEqualChunk(t *testing.T, expected, actual *filer_pb.FileChunk) { assert.Equal(t, expected.FileId, actual.FileId) assert.Equal(t, expected.IsChunkManifest, actual.IsChunkManifest) } func mockMerge(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer_pb.FileChunk) (manifestChunk *filer_pb.FileChunk, err error) { var buf bytes.Buffer minOffset, maxOffset := int64(math.MaxInt64), int64(math.MinInt64) for k := 0; k < len(dataChunks); k++ { chunk := dataChunks[k] buf.WriteString(chunk.FileId) if minOffset > int64(chunk.Offset) { minOffset = chunk.Offset } if maxOffset < int64(chunk.Size)+chunk.Offset { maxOffset = int64(chunk.Size) + chunk.Offset } } manifestChunk = &filer_pb.FileChunk{ FileId: buf.String(), } manifestChunk.IsChunkManifest = true manifestChunk.Offset = minOffset manifestChunk.Size = uint64(maxOffset - minOffset) return } // --------------------------------------------------------------------------- // In-memory manifest store for round-trip tests // --------------------------------------------------------------------------- // testManifestStore provides in-memory save/resolve for manifest tests // without requiring volume servers. type testManifestStore struct { stored map[string][]byte nextID int } func newTestManifestStore() *testManifestStore { return &testManifestStore{stored: make(map[string][]byte)} } func (s *testManifestStore) saveFunc() SaveDataAsChunkFunctionType { return func(reader io.Reader, name string, offset int64, tsNs int64, _ uint64) (*filer_pb.FileChunk, error) { data, err := io.ReadAll(reader) if err != nil { return nil, err } s.nextID++ chunk := &filer_pb.FileChunk{ Fid: &filer_pb.FileId{VolumeId: 999, FileKey: uint64(s.nextID), Cookie: 0}, } s.stored[chunk.GetFileIdString()] = data return chunk, nil } } // resolve expands a single manifest chunk by deserializing the stored proto. func (s *testManifestStore) resolve(chunk *filer_pb.FileChunk) ([]*filer_pb.FileChunk, error) { data, ok := s.stored[chunk.GetFileIdString()] if !ok { return nil, fmt.Errorf("manifest %q not in store", chunk.GetFileIdString()) } m := &filer_pb.FileChunkManifest{} if err := proto.Unmarshal(data, m); err != nil { return nil, err } filer_pb.AfterEntryDeserialization(m.Chunks) return m.Chunks, nil } // resolveAll expands all manifest chunks, returns (dataChunks, manifestChunks). func (s *testManifestStore) resolveAll(chunks []*filer_pb.FileChunk) (data, manifests []*filer_pb.FileChunk, err error) { for _, c := range chunks { if !c.IsChunkManifest { data = append(data, c) continue } manifests = append(manifests, c) sub, resolveErr := s.resolve(c) if resolveErr != nil { return nil, nil, resolveErr } data = append(data, sub...) } return } func testChunk(volId uint32, key uint64, cookie uint32, offset int64, size uint64, tsNs int64) *filer_pb.FileChunk { return &filer_pb.FileChunk{ Fid: &filer_pb.FileId{VolumeId: volId, FileKey: key, Cookie: cookie}, Offset: offset, Size: size, ModifiedTsNs: tsNs, } } // --------------------------------------------------------------------------- // Manifest round-trip: create -> serialize -> deserialize -> verify // --------------------------------------------------------------------------- func TestManifestRoundTripPreservesChunks(t *testing.T) { store := newTestManifestStore() chunks := []*filer_pb.FileChunk{ testChunk(1, 1, 100, 0, 1000, 10), testChunk(1, 2, 200, 1000, 1000, 20), testChunk(2, 3, 300, 2000, 1000, 30), } origIds := make([]string, len(chunks)) for i, c := range chunks { origIds[i] = c.GetFileIdString() } // Use real mergeIntoManifest with batch=3 so all go into one manifest result, err := doMaybeManifestize(store.saveFunc(), chunks, 3, mergeIntoManifest) if err != nil { t.Fatal(err) } if len(result) != 1 || !result[0].IsChunkManifest { t.Fatalf("expected 1 manifest chunk, got %d chunks", len(result)) } if result[0].Offset != 0 || result[0].Size != 3000 { t.Errorf("manifest coverage: offset=%d size=%d, want 0/3000", result[0].Offset, result[0].Size) } resolved, err := store.resolve(result[0]) if err != nil { t.Fatal(err) } if len(resolved) != 3 { t.Fatalf("expected 3 sub-chunks, got %d", len(resolved)) } for i, got := range resolved { if got.GetFileIdString() != origIds[i] { t.Errorf("chunk %d: fileId %q != %q", i, got.GetFileIdString(), origIds[i]) } if got.Offset != chunks[i].Offset { t.Errorf("chunk %d: offset %d != %d", i, got.Offset, chunks[i].Offset) } if got.Size != chunks[i].Size { t.Errorf("chunk %d: size %d != %d", i, got.Size, chunks[i].Size) } if got.ModifiedTsNs != chunks[i].ModifiedTsNs { t.Errorf("chunk %d: ts %d != %d", i, got.ModifiedTsNs, chunks[i].ModifiedTsNs) } } } // --------------------------------------------------------------------------- // Compact resolved overlapping manifests: older sub-chunks become garbage // --------------------------------------------------------------------------- func TestCompactResolvedOverlappingManifests(t *testing.T) { store := newTestManifestStore() // Batch 1: older writes covering [0, 3000) batch1 := []*filer_pb.FileChunk{ testChunk(1, 1, 1, 0, 1000, 1), testChunk(1, 2, 2, 1000, 1000, 2), testChunk(1, 3, 3, 2000, 1000, 3), } origBatch1Ids := make(map[string]bool) for _, c := range batch1 { origBatch1Ids[c.GetFileIdString()] = true } // Batch 2: newer writes covering [0, 3000) batch2 := []*filer_pb.FileChunk{ testChunk(2, 1, 1, 0, 1000, 10), testChunk(2, 2, 2, 1000, 1000, 11), testChunk(2, 3, 3, 2000, 1000, 12), } origBatch2Ids := make(map[string]bool) for _, c := range batch2 { origBatch2Ids[c.GetFileIdString()] = true } // Manifestize each batch separately (simulates successive flush cycles) save := store.saveFunc() manifest1, err := doMaybeManifestize(save, batch1, 3, mergeIntoManifest) if err != nil { t.Fatal(err) } manifest2, err := doMaybeManifestize(save, batch2, 3, mergeIntoManifest) if err != nil { t.Fatal(err) } // Resolve all manifests into sub-chunks allChunks := append(manifest1, manifest2...) dataChunks, _, err := store.resolveAll(allChunks) if err != nil { t.Fatal(err) } if len(dataChunks) != 6 { t.Fatalf("expected 6 resolved data chunks, got %d", len(dataChunks)) } // CompactFileChunks on resolved sub-chunks (nil lookupFn is fine for non-manifest chunks) compacted, garbage := CompactFileChunks(context.Background(), nil, dataChunks) if len(compacted) != 3 { t.Fatalf("expected 3 compacted chunks, got %d", len(compacted)) } if len(garbage) != 3 { t.Fatalf("expected 3 garbage chunks, got %d", len(garbage)) } for _, c := range compacted { if !origBatch2Ids[c.GetFileIdString()] { t.Errorf("compacted chunk %q should be from newer batch", c.GetFileIdString()) } } for _, c := range garbage { if !origBatch1Ids[c.GetFileIdString()] { t.Errorf("garbage chunk %q should be from older batch", c.GetFileIdString()) } } } // --------------------------------------------------------------------------- // DoMinusChunks: old manifest sub-chunks identified as garbage vs new chunks // --------------------------------------------------------------------------- func TestDoMinusChunksWithResolvedManifests(t *testing.T) { store := newTestManifestStore() // Old entry: data packed into a manifest oldData := []*filer_pb.FileChunk{ testChunk(1, 1, 1, 0, 1000, 1), testChunk(1, 2, 2, 1000, 1000, 2), testChunk(1, 3, 3, 2000, 1000, 3), } oldManifest, err := doMaybeManifestize(store.saveFunc(), oldData, 3, mergeIntoManifest) if err != nil { t.Fatal(err) } // Resolve old manifests into sub-chunks (what MinusChunks does internally) oldResolved, oldMeta, err := store.resolveAll(oldManifest) if err != nil { t.Fatal(err) } // New entry: clean re-uploaded chunks (from merge) with different file IDs newData := []*filer_pb.FileChunk{ testChunk(3, 10, 10, 0, 1500, 100), testChunk(3, 11, 11, 1500, 1500, 101), } // DoMinusChunks: data sub-chunks in old but not in new deltaData := DoMinusChunks(oldResolved, newData) if len(deltaData) != 3 { t.Fatalf("expected 3 old data chunks as garbage, got %d", len(deltaData)) } // DoMinusChunks: manifest chunks in old but not in new deltaMeta := DoMinusChunks(oldMeta, nil) if len(deltaMeta) != 1 { t.Fatalf("expected 1 old manifest chunk as garbage, got %d", len(deltaMeta)) } } // --------------------------------------------------------------------------- // Small-batch manifestize: verify sub-chunks and remainder handling // --------------------------------------------------------------------------- func TestManifestizeSmallBatchWithRemainder(t *testing.T) { store := newTestManifestStore() // 7 chunks with batch=3: should produce 2 manifests + 1 remainder chunks := make([]*filer_pb.FileChunk, 7) for i := range chunks { chunks[i] = testChunk(1, uint64(i+1), uint32(i+1), int64(i*100), 100, int64(i+1)) } result, err := doMaybeManifestize(store.saveFunc(), chunks, 3, mergeIntoManifest) if err != nil { t.Fatal(err) } manifests := 0 regular := 0 for _, c := range result { if c.IsChunkManifest { manifests++ } else { regular++ } } if manifests != 2 { t.Errorf("expected 2 manifests, got %d", manifests) } if regular != 1 { t.Errorf("expected 1 remainder chunk, got %d", regular) } // Resolve all and verify we get back 7 data chunks data, _, err := store.resolveAll(result) if err != nil { t.Fatal(err) } if len(data) != 7 { t.Fatalf("expected 7 data chunks after resolve, got %d", len(data)) } } // --------------------------------------------------------------------------- // Multiple overlapping manifests: compaction identifies all redundant sub-chunks // --------------------------------------------------------------------------- func TestCompactMultipleOverlappingManifestGenerations(t *testing.T) { store := newTestManifestStore() save := store.saveFunc() const fileSize int64 = 3000 const chunkSize uint64 = 1000 // Simulate 5 generations of full-file writes, each manifestized separately. var allManifests []*filer_pb.FileChunk for gen := 0; gen < 5; gen++ { tsBase := int64((gen + 1) * 100) batch := []*filer_pb.FileChunk{ testChunk(uint32(gen+1), 1, uint32(gen), 0, chunkSize, tsBase), testChunk(uint32(gen+1), 2, uint32(gen), int64(chunkSize), chunkSize, tsBase+1), testChunk(uint32(gen+1), 3, uint32(gen), int64(chunkSize*2), chunkSize, tsBase+2), } manifest, err := doMaybeManifestize(save, batch, 3, mergeIntoManifest) if err != nil { t.Fatal(err) } allManifests = append(allManifests, manifest...) } if len(allManifests) != 5 { t.Fatalf("expected 5 manifest chunks, got %d", len(allManifests)) } // Resolve all 5 manifests: 15 data chunks total dataChunks, _, err := store.resolveAll(allManifests) if err != nil { t.Fatal(err) } if len(dataChunks) != 15 { t.Fatalf("expected 15 resolved chunks, got %d", len(dataChunks)) } // Compact: only generation 5 (the newest 3 chunks) should survive compacted, garbage := CompactFileChunks(context.Background(), nil, dataChunks) if len(compacted) != 3 { t.Errorf("expected 3 compacted (gen 5), got %d", len(compacted)) } if len(garbage) != 12 { t.Errorf("expected 12 garbage (gens 1-4), got %d", len(garbage)) } // Verify the surviving chunks are the newest (highest timestamps) for _, c := range compacted { if c.ModifiedTsNs < 500 { t.Errorf("compacted chunk ts=%d should be from gen 5 (ts >= 500)", c.ModifiedTsNs) } } } // --------------------------------------------------------------------------- // Bloat detection: manifest sizes must count toward total // --------------------------------------------------------------------------- func TestManifestBloatDetection(t *testing.T) { // Simulates what shouldMergeChunks sees after multiple flush cycles. // With the fix, manifest sizes count toward totalChunkSize, so bloat // from accumulated manifests is detected. const fileSize uint64 = 10000 // 1 regular chunk covering the file (from latest compaction) regular := []*filer_pb.FileChunk{ {Offset: 0, Size: fileSize, FileId: "latest", ModifiedTsNs: 100, Fid: &filer_pb.FileId{VolumeId: 1, FileKey: 1}}, } // N manifests each covering the full file (from prior flush cycles) for n := 1; n <= 10; n++ { var manifests []*filer_pb.FileChunk for i := 0; i < n; i++ { manifests = append(manifests, &filer_pb.FileChunk{ Offset: 0, Size: fileSize, IsChunkManifest: true, Fid: &filer_pb.FileId{VolumeId: 900, FileKey: uint64(i + 1)}, }) } allChunks := append(append([]*filer_pb.FileChunk{}, regular...), manifests...) var totalStored uint64 for _, c := range allChunks { totalStored += c.Size } totalFile := TotalSize(allChunks) expectMerge := totalStored > 2*totalFile t.Logf("n=%d manifests: totalStored=%d fileSize=%d ratio=%.1fx expectMerge=%v", n, totalStored, totalFile, float64(totalStored)/float64(totalFile), expectMerge) // With 1 regular (10000) + N manifests (N*10000): // total = (N+1)*10000, fileSize=10000, ratio = N+1 // Merge when N+1 > 2, i.e., N >= 2 if n >= 2 && !expectMerge { t.Errorf("n=%d: should trigger merge at ratio %.1f", n, float64(totalStored)/float64(totalFile)) } if n < 2 && expectMerge { t.Errorf("n=%d: should NOT trigger merge at ratio %.1f", n, float64(totalStored)/float64(totalFile)) } } } // countingInvalidator stands in for the vidMap cache a mount or filer hands to // manifest resolution, recording how often the locations were dropped. type countingInvalidator struct { invalidations atomic.Int32 } func (inv *countingInvalidator) InvalidateCache(fileId string) { inv.invalidations.Add(1) } // stagedLookup returns staleUrls once and freshUrls afterwards, the way a // vidMapClient re-reads the master once InvalidateCache has dropped the entry. type stagedLookup struct { staleUrls []string freshUrls []string calls atomic.Int32 } func (l *stagedLookup) lookup(ctx context.Context, fileId string) ([]string, error) { if l.calls.Add(1) == 1 { return l.staleUrls, nil } return l.freshUrls, nil } func manifestServer(t *testing.T, manifestBytes []byte) *httptest.Server { t.Helper() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Length", strconv.Itoa(len(manifestBytes))) w.Write(manifestBytes) })) t.Cleanup(srv.Close) return srv } func fetchManifestBuffer(t *testing.T) *bytes.Buffer { t.Helper() bytesBuffer := bytesBufferPool.Get().(*bytes.Buffer) bytesBuffer.Reset() t.Cleanup(func() { bytesBufferPool.Put(bytesBuffer) }) return bytesBuffer } // TestFetchWholeChunkRetriesFreshLocations covers the mount reading a multipart // file whose manifest volume has moved: the cached location is dead, so the // fetch has to drop it and come back with what the master knows now. The stale // server streams a prefix before dying, which the retry must not keep. func TestFetchWholeChunkRetriesFreshLocations(t *testing.T) { manifestBytes, err := proto.Marshal(&filer_pb.FileChunkManifest{ Chunks: []*filer_pb.FileChunk{ {FileId: "100,abc", Offset: 0, Size: 8}, {FileId: "101,def", Offset: 8, Size: 8}, }, }) assert.NoError(t, err) staleSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Length", strconv.Itoa(len(manifestBytes)+64)) w.Write([]byte("garbage-from-stale-server")) w.(http.Flusher).Flush() panic(http.ErrAbortHandler) })) defer staleSrv.Close() lookup := &stagedLookup{ staleUrls: []string{staleSrv.URL + "/5,stale"}, freshUrls: []string{manifestServer(t, manifestBytes).URL + "/5,stale"}, } inv := &countingInvalidator{} bytesBuffer := fetchManifestBuffer(t) assert.NoError(t, fetchWholeChunk(context.Background(), bytesBuffer, lookup.lookup, "5,stale", nil, false, inv)) assert.Equal(t, int32(1), inv.invalidations.Load()) assert.Equal(t, int32(2), lookup.calls.Load()) // the buffer must hold the fresh manifest alone, not the stale prefix ahead of it decoded := &filer_pb.FileChunkManifest{} assert.NoError(t, proto.Unmarshal(bytesBuffer.Bytes(), decoded)) assertEqualChunks(t, []*filer_pb.FileChunk{ {FileId: "100,abc", Offset: 0, Size: 8}, {FileId: "101,def", Offset: 8, Size: 8}, }, decoded.Chunks) } // TestFetchWholeChunkWithoutInvalidator keeps the old behavior for callers whose // lookup function has no cache to drop: the fetch error surfaces as it is. func TestFetchWholeChunkWithoutInvalidator(t *testing.T) { failSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) })) defer failSrv.Close() lookup := &stagedLookup{ staleUrls: []string{failSrv.URL + "/5,abc"}, freshUrls: []string{"http://unused:8080/5,abc"}, } assert.Error(t, fetchWholeChunk(context.Background(), fetchManifestBuffer(t), lookup.lookup, "5,abc", nil, false, nil)) assert.Equal(t, int32(1), lookup.calls.Load()) } // TestFetchWholeChunkUnchangedLocations guards against looping: when the master // still reports the servers that just failed, there is nothing new to try. func TestFetchWholeChunkUnchangedLocations(t *testing.T) { failSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) })) defer failSrv.Close() lookup := &stagedLookup{ staleUrls: []string{failSrv.URL + "/5,abc"}, freshUrls: []string{failSrv.URL + "/5,abc"}, } inv := &countingInvalidator{} assert.Error(t, fetchWholeChunk(context.Background(), fetchManifestBuffer(t), lookup.lookup, "5,abc", nil, false, inv)) assert.Equal(t, int32(2), lookup.calls.Load()) assert.Equal(t, int32(1), inv.invalidations.Load()) } // TestFetchWholeChunkCancelledKeepsLocations checks that a caller walking away // mid-read does not cost every other reader a master round trip. func TestFetchWholeChunkCancelledKeepsLocations(t *testing.T) { lookup := &stagedLookup{ staleUrls: []string{"http://unused:8080/5,abc"}, freshUrls: []string{"http://elsewhere:8080/5,abc"}, } inv := &countingInvalidator{} ctx, cancel := context.WithCancel(context.Background()) cancel() err := fetchWholeChunk(ctx, fetchManifestBuffer(t), lookup.lookup, "5,abc", nil, false, inv) assert.ErrorIs(t, err, context.Canceled) assert.Equal(t, int32(0), inv.invalidations.Load()) assert.Equal(t, int32(1), lookup.calls.Load()) // the cancellation must survive the wrapping ResolveOneChunkManifest does, // or callers cannot tell a dead volume server from their own abort manifestChunk := &filer_pb.FileChunk{FileId: "5,abc", IsChunkManifest: true} _, resolveErr := ResolveOneChunkManifest(ctx, lookup.lookup, manifestChunk, inv) assert.ErrorIs(t, resolveErr, context.Canceled) // volume.fsck resolves manifests with no invalidator and still has to tell // its own abort from a corrupt manifest, so the nil path keeps the identity noInvalidator := retryFetchWithFreshLocations(ctx, nil, lookup.lookup, "5,abc", nil, fmt.Errorf("stale server said no"), func([]string) error { t.Fatal("refetch must not run on a cancelled read") return nil }) assert.ErrorIs(t, noInvalidator, context.Canceled) }