Files
seaweedfs/weed/filer/filechunk_manifest_test.go
T
Chris Lu fe98520358 read: try a replica that stopped answering last, and relearn its volume's locations (#11130)
* http: try a volume server that failed to answer last

A cached location list is shuffled on every read, so once a replica dies
half the reads keep dialing it first and pay a connect failure or timeout
before the healthy replica answers. Remember, per host, when a request got
no answer at all and order such hosts last for the next half minute. Once
that passes, one read probes the host in its usual place while the others
keep it last until the probe settles, so a black-holed server costs one
stalled read per interval instead of one per read.

Nothing is ever skipped: a host that failed is still tried when the others
fail too. Any response, including an error status, counts as reachable.

Claude-Session: https://claude.ai/code/session_011NYXuzGttwrTMsfLYvmQFs

* filer: refresh a chunk's locations after one of them fails

A mount's location cache is only relearned when every cached location
fails. When one replica dies and the other still answers, every read
succeeds and the dead replica stays in the cache, and in the shuffled
order it keeps being dialed first long after the master has dropped it.

When a read fails on one location and a later one answers, call the
refresh hook so the cached entry is dropped and looked up again. The read
that already paid for the failure returns its data; the reads after it
start from the locations the master knows now.

Claude-Session: https://claude.ai/code/session_011NYXuzGttwrTMsfLYvmQFs

* http: claim the probe for every expired host, and try it first

The claim was only checked for the first url, so with two replicas whose
marks expired together the second was probed by every read at once. Claim
each expired host on its own and put the reads that won a claim ahead of
the reachable hosts, so a probe is always a real attempt and a lost claim
always means the host is tried last.

Claude-Session: https://claude.ai/code/session_011NYXuzGttwrTMsfLYvmQFs

* filer: refresh a chunk's locations in the streaming read path too

The streaming loop had no refresh hook, so a manifest or streamed chunk
that failed on one cached location and was served by another kept the
stale entry until every location failed. Give it the same hook as the
buffered loop, built by one refreshUrls function shared by the reader
cache and the stream callers.

Claude-Session: https://claude.ai/code/session_011NYXuzGttwrTMsfLYvmQFs

* http: probe at most one expired host per read

Claiming every expired host in one ordering left all but the first claim
without an attempt, since a read stops at its first answer, and a host that
had come back waited another interval for nothing. Claim only the first
expired host a read sees and leave the rest last and unclaimed, so each
following read probes one of them.

Claude-Session: https://claude.ai/code/session_011NYXuzGttwrTMsfLYvmQFs

* test: start the live server before releasing the dead server's port

Closing the dead server first let the live server come up on the same
port, in which case the dead location answers and the partial failure
under test never happens.

Claude-Session: https://claude.ai/code/session_011NYXuzGttwrTMsfLYvmQFs
2026-09-03 11:51:48 -07:00

692 lines
22 KiB
Go

package filer
import (
"bytes"
"context"
"fmt"
"io"
"math"
"net/http"
"net/http/httptest"
"strconv"
"sync/atomic"
"testing"
"github.com/stretchr/testify/assert"
"google.golang.org/protobuf/proto"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func TestDoMaybeManifestize(t *testing.T) {
var manifestTests = []struct {
inputs []*filer_pb.FileChunk
expected []*filer_pb.FileChunk
}{
{
inputs: []*filer_pb.FileChunk{
{FileId: "1", IsChunkManifest: false},
{FileId: "2", IsChunkManifest: false},
{FileId: "3", IsChunkManifest: false},
{FileId: "4", IsChunkManifest: false},
},
expected: []*filer_pb.FileChunk{
{FileId: "12", IsChunkManifest: true},
{FileId: "34", IsChunkManifest: true},
},
},
{
inputs: []*filer_pb.FileChunk{
{FileId: "1", IsChunkManifest: true},
{FileId: "2", IsChunkManifest: false},
{FileId: "3", IsChunkManifest: false},
{FileId: "4", IsChunkManifest: false},
},
expected: []*filer_pb.FileChunk{
{FileId: "1", IsChunkManifest: true},
{FileId: "23", IsChunkManifest: true},
{FileId: "4", IsChunkManifest: false},
},
},
{
inputs: []*filer_pb.FileChunk{
{FileId: "1", IsChunkManifest: false},
{FileId: "2", IsChunkManifest: true},
{FileId: "3", IsChunkManifest: false},
{FileId: "4", IsChunkManifest: false},
},
expected: []*filer_pb.FileChunk{
{FileId: "2", IsChunkManifest: true},
{FileId: "13", IsChunkManifest: true},
{FileId: "4", IsChunkManifest: false},
},
},
{
inputs: []*filer_pb.FileChunk{
{FileId: "1", IsChunkManifest: true},
{FileId: "2", IsChunkManifest: true},
{FileId: "3", IsChunkManifest: false},
{FileId: "4", IsChunkManifest: false},
},
expected: []*filer_pb.FileChunk{
{FileId: "1", IsChunkManifest: true},
{FileId: "2", IsChunkManifest: true},
{FileId: "34", IsChunkManifest: true},
},
},
}
for i, mtest := range manifestTests {
println("test", i)
actual, _ := doMaybeManifestize(nil, mtest.inputs, 2, mockMerge)
assertEqualChunks(t, mtest.expected, actual)
}
}
func assertEqualChunks(t *testing.T, expected, actual []*filer_pb.FileChunk) {
assert.Equal(t, len(expected), len(actual))
for i := 0; i < len(actual); i++ {
assertEqualChunk(t, actual[i], expected[i])
}
}
func assertEqualChunk(t *testing.T, expected, actual *filer_pb.FileChunk) {
assert.Equal(t, expected.FileId, actual.FileId)
assert.Equal(t, expected.IsChunkManifest, actual.IsChunkManifest)
}
func mockMerge(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer_pb.FileChunk) (manifestChunk *filer_pb.FileChunk, err error) {
var buf bytes.Buffer
minOffset, maxOffset := int64(math.MaxInt64), int64(math.MinInt64)
for k := 0; k < len(dataChunks); k++ {
chunk := dataChunks[k]
buf.WriteString(chunk.FileId)
if minOffset > int64(chunk.Offset) {
minOffset = chunk.Offset
}
if maxOffset < int64(chunk.Size)+chunk.Offset {
maxOffset = int64(chunk.Size) + chunk.Offset
}
}
manifestChunk = &filer_pb.FileChunk{
FileId: buf.String(),
}
manifestChunk.IsChunkManifest = true
manifestChunk.Offset = minOffset
manifestChunk.Size = uint64(maxOffset - minOffset)
return
}
// ---------------------------------------------------------------------------
// In-memory manifest store for round-trip tests
// ---------------------------------------------------------------------------
// testManifestStore provides in-memory save/resolve for manifest tests
// without requiring volume servers.
type testManifestStore struct {
stored map[string][]byte
nextID int
}
func newTestManifestStore() *testManifestStore {
return &testManifestStore{stored: make(map[string][]byte)}
}
func (s *testManifestStore) saveFunc() SaveDataAsChunkFunctionType {
return func(reader io.Reader, name string, offset int64, tsNs int64, _ uint64) (*filer_pb.FileChunk, error) {
data, err := io.ReadAll(reader)
if err != nil {
return nil, err
}
s.nextID++
chunk := &filer_pb.FileChunk{
Fid: &filer_pb.FileId{VolumeId: 999, FileKey: uint64(s.nextID), Cookie: 0},
}
s.stored[chunk.GetFileIdString()] = data
return chunk, nil
}
}
// resolve expands a single manifest chunk by deserializing the stored proto.
func (s *testManifestStore) resolve(chunk *filer_pb.FileChunk) ([]*filer_pb.FileChunk, error) {
data, ok := s.stored[chunk.GetFileIdString()]
if !ok {
return nil, fmt.Errorf("manifest %q not in store", chunk.GetFileIdString())
}
m := &filer_pb.FileChunkManifest{}
if err := proto.Unmarshal(data, m); err != nil {
return nil, err
}
filer_pb.AfterEntryDeserialization(m.Chunks)
return m.Chunks, nil
}
// resolveAll expands all manifest chunks, returns (dataChunks, manifestChunks).
func (s *testManifestStore) resolveAll(chunks []*filer_pb.FileChunk) (data, manifests []*filer_pb.FileChunk, err error) {
for _, c := range chunks {
if !c.IsChunkManifest {
data = append(data, c)
continue
}
manifests = append(manifests, c)
sub, resolveErr := s.resolve(c)
if resolveErr != nil {
return nil, nil, resolveErr
}
data = append(data, sub...)
}
return
}
func testChunk(volId uint32, key uint64, cookie uint32, offset int64, size uint64, tsNs int64) *filer_pb.FileChunk {
return &filer_pb.FileChunk{
Fid: &filer_pb.FileId{VolumeId: volId, FileKey: key, Cookie: cookie},
Offset: offset,
Size: size,
ModifiedTsNs: tsNs,
}
}
// ---------------------------------------------------------------------------
// Manifest round-trip: create -> serialize -> deserialize -> verify
// ---------------------------------------------------------------------------
func TestManifestRoundTripPreservesChunks(t *testing.T) {
store := newTestManifestStore()
chunks := []*filer_pb.FileChunk{
testChunk(1, 1, 100, 0, 1000, 10),
testChunk(1, 2, 200, 1000, 1000, 20),
testChunk(2, 3, 300, 2000, 1000, 30),
}
origIds := make([]string, len(chunks))
for i, c := range chunks {
origIds[i] = c.GetFileIdString()
}
// Use real mergeIntoManifest with batch=3 so all go into one manifest
result, err := doMaybeManifestize(store.saveFunc(), chunks, 3, mergeIntoManifest)
if err != nil {
t.Fatal(err)
}
if len(result) != 1 || !result[0].IsChunkManifest {
t.Fatalf("expected 1 manifest chunk, got %d chunks", len(result))
}
if result[0].Offset != 0 || result[0].Size != 3000 {
t.Errorf("manifest coverage: offset=%d size=%d, want 0/3000", result[0].Offset, result[0].Size)
}
resolved, err := store.resolve(result[0])
if err != nil {
t.Fatal(err)
}
if len(resolved) != 3 {
t.Fatalf("expected 3 sub-chunks, got %d", len(resolved))
}
for i, got := range resolved {
if got.GetFileIdString() != origIds[i] {
t.Errorf("chunk %d: fileId %q != %q", i, got.GetFileIdString(), origIds[i])
}
if got.Offset != chunks[i].Offset {
t.Errorf("chunk %d: offset %d != %d", i, got.Offset, chunks[i].Offset)
}
if got.Size != chunks[i].Size {
t.Errorf("chunk %d: size %d != %d", i, got.Size, chunks[i].Size)
}
if got.ModifiedTsNs != chunks[i].ModifiedTsNs {
t.Errorf("chunk %d: ts %d != %d", i, got.ModifiedTsNs, chunks[i].ModifiedTsNs)
}
}
}
// ---------------------------------------------------------------------------
// Compact resolved overlapping manifests: older sub-chunks become garbage
// ---------------------------------------------------------------------------
func TestCompactResolvedOverlappingManifests(t *testing.T) {
store := newTestManifestStore()
// Batch 1: older writes covering [0, 3000)
batch1 := []*filer_pb.FileChunk{
testChunk(1, 1, 1, 0, 1000, 1),
testChunk(1, 2, 2, 1000, 1000, 2),
testChunk(1, 3, 3, 2000, 1000, 3),
}
origBatch1Ids := make(map[string]bool)
for _, c := range batch1 {
origBatch1Ids[c.GetFileIdString()] = true
}
// Batch 2: newer writes covering [0, 3000)
batch2 := []*filer_pb.FileChunk{
testChunk(2, 1, 1, 0, 1000, 10),
testChunk(2, 2, 2, 1000, 1000, 11),
testChunk(2, 3, 3, 2000, 1000, 12),
}
origBatch2Ids := make(map[string]bool)
for _, c := range batch2 {
origBatch2Ids[c.GetFileIdString()] = true
}
// Manifestize each batch separately (simulates successive flush cycles)
save := store.saveFunc()
manifest1, err := doMaybeManifestize(save, batch1, 3, mergeIntoManifest)
if err != nil {
t.Fatal(err)
}
manifest2, err := doMaybeManifestize(save, batch2, 3, mergeIntoManifest)
if err != nil {
t.Fatal(err)
}
// Resolve all manifests into sub-chunks
allChunks := append(manifest1, manifest2...)
dataChunks, _, err := store.resolveAll(allChunks)
if err != nil {
t.Fatal(err)
}
if len(dataChunks) != 6 {
t.Fatalf("expected 6 resolved data chunks, got %d", len(dataChunks))
}
// CompactFileChunks on resolved sub-chunks (nil lookupFn is fine for non-manifest chunks)
compacted, garbage := CompactFileChunks(context.Background(), nil, dataChunks)
if len(compacted) != 3 {
t.Fatalf("expected 3 compacted chunks, got %d", len(compacted))
}
if len(garbage) != 3 {
t.Fatalf("expected 3 garbage chunks, got %d", len(garbage))
}
for _, c := range compacted {
if !origBatch2Ids[c.GetFileIdString()] {
t.Errorf("compacted chunk %q should be from newer batch", c.GetFileIdString())
}
}
for _, c := range garbage {
if !origBatch1Ids[c.GetFileIdString()] {
t.Errorf("garbage chunk %q should be from older batch", c.GetFileIdString())
}
}
}
// ---------------------------------------------------------------------------
// DoMinusChunks: old manifest sub-chunks identified as garbage vs new chunks
// ---------------------------------------------------------------------------
func TestDoMinusChunksWithResolvedManifests(t *testing.T) {
store := newTestManifestStore()
// Old entry: data packed into a manifest
oldData := []*filer_pb.FileChunk{
testChunk(1, 1, 1, 0, 1000, 1),
testChunk(1, 2, 2, 1000, 1000, 2),
testChunk(1, 3, 3, 2000, 1000, 3),
}
oldManifest, err := doMaybeManifestize(store.saveFunc(), oldData, 3, mergeIntoManifest)
if err != nil {
t.Fatal(err)
}
// Resolve old manifests into sub-chunks (what MinusChunks does internally)
oldResolved, oldMeta, err := store.resolveAll(oldManifest)
if err != nil {
t.Fatal(err)
}
// New entry: clean re-uploaded chunks (from merge) with different file IDs
newData := []*filer_pb.FileChunk{
testChunk(3, 10, 10, 0, 1500, 100),
testChunk(3, 11, 11, 1500, 1500, 101),
}
// DoMinusChunks: data sub-chunks in old but not in new
deltaData := DoMinusChunks(oldResolved, newData)
if len(deltaData) != 3 {
t.Fatalf("expected 3 old data chunks as garbage, got %d", len(deltaData))
}
// DoMinusChunks: manifest chunks in old but not in new
deltaMeta := DoMinusChunks(oldMeta, nil)
if len(deltaMeta) != 1 {
t.Fatalf("expected 1 old manifest chunk as garbage, got %d", len(deltaMeta))
}
}
// ---------------------------------------------------------------------------
// Small-batch manifestize: verify sub-chunks and remainder handling
// ---------------------------------------------------------------------------
func TestManifestizeSmallBatchWithRemainder(t *testing.T) {
store := newTestManifestStore()
// 7 chunks with batch=3: should produce 2 manifests + 1 remainder
chunks := make([]*filer_pb.FileChunk, 7)
for i := range chunks {
chunks[i] = testChunk(1, uint64(i+1), uint32(i+1), int64(i*100), 100, int64(i+1))
}
result, err := doMaybeManifestize(store.saveFunc(), chunks, 3, mergeIntoManifest)
if err != nil {
t.Fatal(err)
}
manifests := 0
regular := 0
for _, c := range result {
if c.IsChunkManifest {
manifests++
} else {
regular++
}
}
if manifests != 2 {
t.Errorf("expected 2 manifests, got %d", manifests)
}
if regular != 1 {
t.Errorf("expected 1 remainder chunk, got %d", regular)
}
// Resolve all and verify we get back 7 data chunks
data, _, err := store.resolveAll(result)
if err != nil {
t.Fatal(err)
}
if len(data) != 7 {
t.Fatalf("expected 7 data chunks after resolve, got %d", len(data))
}
}
// ---------------------------------------------------------------------------
// Multiple overlapping manifests: compaction identifies all redundant sub-chunks
// ---------------------------------------------------------------------------
func TestCompactMultipleOverlappingManifestGenerations(t *testing.T) {
store := newTestManifestStore()
save := store.saveFunc()
const fileSize int64 = 3000
const chunkSize uint64 = 1000
// Simulate 5 generations of full-file writes, each manifestized separately.
var allManifests []*filer_pb.FileChunk
for gen := 0; gen < 5; gen++ {
tsBase := int64((gen + 1) * 100)
batch := []*filer_pb.FileChunk{
testChunk(uint32(gen+1), 1, uint32(gen), 0, chunkSize, tsBase),
testChunk(uint32(gen+1), 2, uint32(gen), int64(chunkSize), chunkSize, tsBase+1),
testChunk(uint32(gen+1), 3, uint32(gen), int64(chunkSize*2), chunkSize, tsBase+2),
}
manifest, err := doMaybeManifestize(save, batch, 3, mergeIntoManifest)
if err != nil {
t.Fatal(err)
}
allManifests = append(allManifests, manifest...)
}
if len(allManifests) != 5 {
t.Fatalf("expected 5 manifest chunks, got %d", len(allManifests))
}
// Resolve all 5 manifests: 15 data chunks total
dataChunks, _, err := store.resolveAll(allManifests)
if err != nil {
t.Fatal(err)
}
if len(dataChunks) != 15 {
t.Fatalf("expected 15 resolved chunks, got %d", len(dataChunks))
}
// Compact: only generation 5 (the newest 3 chunks) should survive
compacted, garbage := CompactFileChunks(context.Background(), nil, dataChunks)
if len(compacted) != 3 {
t.Errorf("expected 3 compacted (gen 5), got %d", len(compacted))
}
if len(garbage) != 12 {
t.Errorf("expected 12 garbage (gens 1-4), got %d", len(garbage))
}
// Verify the surviving chunks are the newest (highest timestamps)
for _, c := range compacted {
if c.ModifiedTsNs < 500 {
t.Errorf("compacted chunk ts=%d should be from gen 5 (ts >= 500)", c.ModifiedTsNs)
}
}
}
// ---------------------------------------------------------------------------
// Bloat detection: manifest sizes must count toward total
// ---------------------------------------------------------------------------
func TestManifestBloatDetection(t *testing.T) {
// Simulates what shouldMergeChunks sees after multiple flush cycles.
// With the fix, manifest sizes count toward totalChunkSize, so bloat
// from accumulated manifests is detected.
const fileSize uint64 = 10000
// 1 regular chunk covering the file (from latest compaction)
regular := []*filer_pb.FileChunk{
{Offset: 0, Size: fileSize, FileId: "latest", ModifiedTsNs: 100,
Fid: &filer_pb.FileId{VolumeId: 1, FileKey: 1}},
}
// N manifests each covering the full file (from prior flush cycles)
for n := 1; n <= 10; n++ {
var manifests []*filer_pb.FileChunk
for i := 0; i < n; i++ {
manifests = append(manifests, &filer_pb.FileChunk{
Offset: 0, Size: fileSize, IsChunkManifest: true,
Fid: &filer_pb.FileId{VolumeId: 900, FileKey: uint64(i + 1)},
})
}
allChunks := append(append([]*filer_pb.FileChunk{}, regular...), manifests...)
var totalStored uint64
for _, c := range allChunks {
totalStored += c.Size
}
totalFile := TotalSize(allChunks)
expectMerge := totalStored > 2*totalFile
t.Logf("n=%d manifests: totalStored=%d fileSize=%d ratio=%.1fx expectMerge=%v",
n, totalStored, totalFile, float64(totalStored)/float64(totalFile), expectMerge)
// With 1 regular (10000) + N manifests (N*10000):
// total = (N+1)*10000, fileSize=10000, ratio = N+1
// Merge when N+1 > 2, i.e., N >= 2
if n >= 2 && !expectMerge {
t.Errorf("n=%d: should trigger merge at ratio %.1f", n, float64(totalStored)/float64(totalFile))
}
if n < 2 && expectMerge {
t.Errorf("n=%d: should NOT trigger merge at ratio %.1f", n, float64(totalStored)/float64(totalFile))
}
}
}
// countingInvalidator stands in for the vidMap cache a mount or filer hands to
// manifest resolution, recording how often the locations were dropped.
type countingInvalidator struct {
invalidations atomic.Int32
}
func (inv *countingInvalidator) InvalidateCache(fileId string) {
inv.invalidations.Add(1)
}
// stagedLookup returns staleUrls once and freshUrls afterwards, the way a
// vidMapClient re-reads the master once InvalidateCache has dropped the entry.
type stagedLookup struct {
staleUrls []string
freshUrls []string
calls atomic.Int32
}
func (l *stagedLookup) lookup(ctx context.Context, fileId string) ([]string, error) {
if l.calls.Add(1) == 1 {
return l.staleUrls, nil
}
return l.freshUrls, nil
}
func manifestServer(t *testing.T, manifestBytes []byte) *httptest.Server {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Length", strconv.Itoa(len(manifestBytes)))
w.Write(manifestBytes)
}))
t.Cleanup(srv.Close)
return srv
}
func fetchManifestBuffer(t *testing.T) *bytes.Buffer {
t.Helper()
bytesBuffer := bytesBufferPool.Get().(*bytes.Buffer)
bytesBuffer.Reset()
t.Cleanup(func() { bytesBufferPool.Put(bytesBuffer) })
return bytesBuffer
}
// TestFetchWholeChunkRetriesFreshLocations covers the mount reading a multipart
// file whose manifest volume has moved: the cached location is dead, so the
// fetch has to drop it and come back with what the master knows now. The stale
// server streams a prefix before dying, which the retry must not keep.
func TestFetchWholeChunkRetriesFreshLocations(t *testing.T) {
manifestBytes, err := proto.Marshal(&filer_pb.FileChunkManifest{
Chunks: []*filer_pb.FileChunk{
{FileId: "100,abc", Offset: 0, Size: 8},
{FileId: "101,def", Offset: 8, Size: 8},
},
})
assert.NoError(t, err)
staleSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Length", strconv.Itoa(len(manifestBytes)+64))
w.Write([]byte("garbage-from-stale-server"))
w.(http.Flusher).Flush()
panic(http.ErrAbortHandler)
}))
defer staleSrv.Close()
lookup := &stagedLookup{
staleUrls: []string{staleSrv.URL + "/5,stale"},
freshUrls: []string{manifestServer(t, manifestBytes).URL + "/5,stale"},
}
inv := &countingInvalidator{}
bytesBuffer := fetchManifestBuffer(t)
assert.NoError(t, fetchWholeChunk(context.Background(), bytesBuffer, lookup.lookup, "5,stale", nil, false, inv))
assert.Equal(t, int32(1), inv.invalidations.Load())
assert.Equal(t, int32(2), lookup.calls.Load())
// the buffer must hold the fresh manifest alone, not the stale prefix ahead of it
decoded := &filer_pb.FileChunkManifest{}
assert.NoError(t, proto.Unmarshal(bytesBuffer.Bytes(), decoded))
assertEqualChunks(t, []*filer_pb.FileChunk{
{FileId: "100,abc", Offset: 0, Size: 8},
{FileId: "101,def", Offset: 8, Size: 8},
}, decoded.Chunks)
}
// TestFetchWholeChunkRefreshesLocationsAfterPartialFailure is the manifest read
// whose cached list still names a dead replica ahead of a live one: the read
// succeeds, and the cached entry must still be dropped and looked up again.
func TestFetchWholeChunkRefreshesLocationsAfterPartialFailure(t *testing.T) {
manifestBytes, err := proto.Marshal(&filer_pb.FileChunkManifest{
Chunks: []*filer_pb.FileChunk{{FileId: "100,abc", Offset: 0, Size: 8}},
})
assert.NoError(t, err)
liveURL := manifestServer(t, manifestBytes).URL + "/5,abc"
dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
deadURL := dead.URL
dead.Close()
lookup := &stagedLookup{
staleUrls: []string{deadURL + "/5,abc", liveURL},
freshUrls: []string{liveURL},
}
inv := &countingInvalidator{}
bytesBuffer := fetchManifestBuffer(t)
assert.NoError(t, fetchWholeChunk(context.Background(), bytesBuffer, lookup.lookup, "5,abc", nil, false, inv))
assert.Equal(t, int32(1), inv.invalidations.Load())
assert.Equal(t, int32(2), lookup.calls.Load())
decoded := &filer_pb.FileChunkManifest{}
assert.NoError(t, proto.Unmarshal(bytesBuffer.Bytes(), decoded))
assertEqualChunks(t, []*filer_pb.FileChunk{{FileId: "100,abc", Offset: 0, Size: 8}}, decoded.Chunks)
}
// TestFetchWholeChunkWithoutInvalidator keeps the old behavior for callers whose
// lookup function has no cache to drop: the fetch error surfaces as it is.
func TestFetchWholeChunkWithoutInvalidator(t *testing.T) {
failSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer failSrv.Close()
lookup := &stagedLookup{
staleUrls: []string{failSrv.URL + "/5,abc"},
freshUrls: []string{"http://unused:8080/5,abc"},
}
assert.Error(t, fetchWholeChunk(context.Background(), fetchManifestBuffer(t), lookup.lookup, "5,abc", nil, false, nil))
assert.Equal(t, int32(1), lookup.calls.Load())
}
// TestFetchWholeChunkUnchangedLocations guards against looping: when the master
// still reports the servers that just failed, there is nothing new to try.
func TestFetchWholeChunkUnchangedLocations(t *testing.T) {
failSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer failSrv.Close()
lookup := &stagedLookup{
staleUrls: []string{failSrv.URL + "/5,abc"},
freshUrls: []string{failSrv.URL + "/5,abc"},
}
inv := &countingInvalidator{}
assert.Error(t, fetchWholeChunk(context.Background(), fetchManifestBuffer(t), lookup.lookup, "5,abc", nil, false, inv))
assert.Equal(t, int32(2), lookup.calls.Load())
assert.Equal(t, int32(1), inv.invalidations.Load())
}
// TestFetchWholeChunkCancelledKeepsLocations checks that a caller walking away
// mid-read does not cost every other reader a master round trip.
func TestFetchWholeChunkCancelledKeepsLocations(t *testing.T) {
lookup := &stagedLookup{
staleUrls: []string{"http://unused:8080/5,abc"},
freshUrls: []string{"http://elsewhere:8080/5,abc"},
}
inv := &countingInvalidator{}
ctx, cancel := context.WithCancel(context.Background())
cancel()
err := fetchWholeChunk(ctx, fetchManifestBuffer(t), lookup.lookup, "5,abc", nil, false, inv)
assert.ErrorIs(t, err, context.Canceled)
assert.Equal(t, int32(0), inv.invalidations.Load())
assert.Equal(t, int32(1), lookup.calls.Load())
// the cancellation must survive the wrapping ResolveOneChunkManifest does,
// or callers cannot tell a dead volume server from their own abort
manifestChunk := &filer_pb.FileChunk{FileId: "5,abc", IsChunkManifest: true}
_, resolveErr := ResolveOneChunkManifest(ctx, lookup.lookup, manifestChunk, inv)
assert.ErrorIs(t, resolveErr, context.Canceled)
// volume.fsck resolves manifests with no invalidator and still has to tell
// its own abort from a corrupt manifest, so the nil path keeps the identity
noInvalidator := retryFetchWithFreshLocations(ctx, nil, lookup.lookup, "5,abc", nil, fmt.Errorf("stale server said no"), func([]string) error {
t.Fatal("refetch must not run on a cancelled read")
return nil
})
assert.ErrorIs(t, noInvalidator, context.Canceled)
}