seaweedfs/weed/filer/persisted_log_cache_test.go

package filer

import (
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)

func logEntriesAt(tsNs ...int64) []*filer_pb.LogEntry {
	out := make([]*filer_pb.LogEntry, 0, len(tsNs))
	for _, ts := range tsNs {
		out = append(out, &filer_pb.LogEntry{TsNs: ts, Data: []byte("x")})
	}
	return out
}

func TestPersistedLogCacheHitMiss(t *testing.T) {
	c := newPersistedLogCache(persistedLogCacheMaxBytes)
	var loads int32
	load := func() ([]*filer_pb.LogEntry, bool, error) {
		atomic.AddInt32(&loads, 1)
		return logEntriesAt(1, 2, 3), true, nil
	}

	e1, err := c.getOrLoad("k", "fp", load)
	if err != nil || len(e1) != 3 {
		t.Fatalf("first getOrLoad: err=%v len=%d", err, len(e1))
	}
	e2, err := c.getOrLoad("k", "fp", load)
	if err != nil {
		t.Fatal(err)
	}
	if n := atomic.LoadInt32(&loads); n != 1 {
		t.Fatalf("expected loader called once, got %d", n)
	}
	if &e1[0] != &e2[0] {
		t.Fatal("cache should return the same shared slice")
	}
}

func TestPersistedLogCacheFingerprintReload(t *testing.T) {
	c := newPersistedLogCache(persistedLogCacheMaxBytes)
	var loads int32
	loadV1 := func() ([]*filer_pb.LogEntry, bool, error) {
		atomic.AddInt32(&loads, 1)
		return logEntriesAt(1, 2), true, nil
	}
	loadV2 := func() ([]*filer_pb.LogEntry, bool, error) {
		atomic.AddInt32(&loads, 1)
		return logEntriesAt(1, 2, 3), true, nil // a delayed append grew the file
	}

	if e, _ := c.getOrLoad("k", "fp1", loadV1); len(e) != 2 {
		t.Fatalf("v1 len=%d", len(e))
	}
	// a later replay observes the grown file (new fingerprint): it must reload,
	// not serve the stale truncated snapshot.
	e, err := c.getOrLoad("k", "fp2", loadV2)
	if err != nil {
		t.Fatal(err)
	}
	if len(e) != 3 {
		t.Fatalf("expected reload to 3 entries, got %d", len(e))
	}
	if n := atomic.LoadInt32(&loads); n != 2 {
		t.Fatalf("expected one reload (2 loads), got %d", n)
	}
	// the cache now holds the new version; the matching fingerprint hits
	if _, err := c.getOrLoad("k", "fp2", loadV2); err != nil {
		t.Fatal(err)
	}
	if n := atomic.LoadInt32(&loads); n != 2 {
		t.Fatalf("expected fp2 to hit (still 2 loads), got %d", n)
	}
}

func TestPersistedLogCacheNotCachedWhenUncacheable(t *testing.T) {
	c := newPersistedLogCache(persistedLogCacheMaxBytes)
	var loads int32
	// cacheable=false models a chunk-not-found stop: deliver the prefix this
	// round but never pin it, so a transient outage is re-probed next replay.
	load := func() ([]*filer_pb.LogEntry, bool, error) {
		atomic.AddInt32(&loads, 1)
		return logEntriesAt(1, 2), false, nil
	}

	if e, err := c.getOrLoad("k", "fp", load); err != nil || len(e) != 2 {
		t.Fatalf("first: err=%v len=%d", err, len(e))
	}
	if e, err := c.getOrLoad("k", "fp", load); err != nil || len(e) != 2 {
		t.Fatalf("second: err=%v len=%d", err, len(e))
	}
	if n := atomic.LoadInt32(&loads); n != 2 {
		t.Fatalf("uncacheable result must re-load every time, got %d loads", n)
	}
}

func TestChunksFingerprintChangesOnGrowth(t *testing.T) {
	c1 := []*filer_pb.FileChunk{{FileId: "3,01", Size: 100}}
	c2 := []*filer_pb.FileChunk{{FileId: "3,01", Size: 100}, {FileId: "3,02", Size: 50}}
	if chunksFingerprint(c1) == chunksFingerprint(c2) {
		t.Fatal("fingerprint must change when a chunk is appended")
	}
}

func TestPersistedLogCacheSingleFlight(t *testing.T) {
	c := newPersistedLogCache(persistedLogCacheMaxBytes)
	var loads int32
	release := make(chan struct{})
	load := func() ([]*filer_pb.LogEntry, bool, error) {
		atomic.AddInt32(&loads, 1)
		<-release // hold the flight open so concurrent callers coalesce
		return logEntriesAt(1), true, nil
	}

	const n = 20
	var wg sync.WaitGroup
	for i := 0; i < n; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			if _, err := c.getOrLoad("k", "fp", load); err != nil {
				t.Error(err)
			}
		}()
	}
	time.Sleep(50 * time.Millisecond)
	close(release)
	wg.Wait()

	if got := atomic.LoadInt32(&loads); got != 1 {
		t.Fatalf("expected 1 coalesced load across %d callers, got %d", n, got)
	}
}

func TestPersistedLogCacheEviction(t *testing.T) {
	// One entry estimates to 128 + 100 + 16 = 244 bytes; budget holds one, not two.
	c := newPersistedLogCache(300)
	mk := func(ts int64) func() ([]*filer_pb.LogEntry, bool, error) {
		return func() ([]*filer_pb.LogEntry, bool, error) {
			return []*filer_pb.LogEntry{{TsNs: ts, Data: make([]byte, 100)}}, true, nil
		}
	}

	if _, err := c.getOrLoad("a", "fp", mk(1)); err != nil {
		t.Fatal(err)
	}
	if _, err := c.getOrLoad("b", "fp", mk(2)); err != nil { // pushes over budget, evicts LRU "a"
		t.Fatal(err)
	}

	c.mu.Lock()
	_, hasA := c.index["a"]
	_, hasB := c.index["b"]
	c.mu.Unlock()
	if hasA {
		t.Error("least-recently-used entry a should have been evicted")
	}
	if !hasB {
		t.Error("most-recently-used entry b should remain")
	}
}

func TestLogFileIsCacheable(t *testing.T) {
	now := time.Now()
	if logFileIsCacheable(now.UnixNano()) {
		t.Error("a current-minute file must not be cacheable")
	}
	old := now.Add(-persistedLogCacheMinAge - time.Minute).UnixNano()
	if !logFileIsCacheable(old) {
		t.Error("a file older than the min age should be cacheable")
	}
}

func TestLogFileIteratorCachedFiltering(t *testing.T) {
	iter := &LogFileIterator{
		cached:    logEntriesAt(10, 20, 30, 40),
		startTsNs: 15,
		stopTsNs:  35,
	}
	var got []int64
	for {
		e, err := iter.getNext()
		if err == io.EOF {
			break
		}
		if err != nil {
			t.Fatal(err)
		}
		got = append(got, e.TsNs)
	}
	want := []int64{20, 30} // 10 filtered by startTsNs, 40 cut off by stopTsNs
	if fmt.Sprint(got) != fmt.Sprint(want) {
		t.Fatalf("cached filtering: got %v, want %v", got, want)
	}
}

func TestLogFileIteratorCachedLoadError(t *testing.T) {
	iter := &LogFileIterator{loadErr: fmt.Errorf("boom")}
	if _, err := iter.getNext(); err == nil || err.Error() != "boom" {
		t.Fatalf("expected load error surfaced on first getNext, got %v", err)
	}
}

func TestLogFileIteratorCachedYieldsBeforeError(t *testing.T) {
	iter := &LogFileIterator{cached: logEntriesAt(10, 20), loadErr: fmt.Errorf("boom")}
	var got []int64
	for {
		e, err := iter.getNext()
		if err != nil {
			if err.Error() != "boom" {
				t.Fatalf("expected boom after entries, got %v", err)
			}
			break
		}
		got = append(got, e.TsNs)
	}
	if fmt.Sprint(got) != fmt.Sprint([]int64{10, 20}) {
		t.Fatalf("partial read must yield entries before the error, got %v", got)
	}
}