seaweedfs/weed/filer/leveldb_handle/leveldb_handle_test.go

package leveldb_handle

import (
	"errors"
	"sync"
	"sync/atomic"
	"syscall"
	"testing"
	"time"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/storage"
)

type failingStorage struct {
	storage.Storage
	fail         atomic.Bool
	failOpen     atomic.Bool
	probeStarted chan struct{}
	probeRelease chan struct{}
}

func (f *failingStorage) Create(fd storage.FileDesc) (storage.Writer, error) {
	if f.failOpen.Load() && fd.Type != storage.TypeTemp {
		return nil, syscall.ENOSPC
	}
	if fd.Type == storage.TypeTemp && f.probeRelease != nil {
		f.probeStarted <- struct{}{}
		<-f.probeRelease
	}
	w, err := f.Storage.Create(fd)
	if err != nil {
		return nil, err
	}
	return &failingWriter{Writer: w, s: f}, nil
}

type failingWriter struct {
	storage.Writer
	s *failingStorage
}

func (w *failingWriter) Write(p []byte) (int, error) {
	if w.s.fail.Load() {
		return 0, syscall.ENOSPC
	}
	return w.Writer.Write(p)
}

func openFailing(t *testing.T) (*DB, *failingStorage) {
	t.Helper()
	interval, busy, timeout, wait := reopenInterval, busyReopenInterval, drainTimeout, reopenWait
	reopenInterval, busyReopenInterval, drainTimeout, reopenWait = 0, 0, 50*time.Millisecond, 50*time.Millisecond
	t.Cleanup(func() {
		reopenInterval, busyReopenInterval, drainTimeout, reopenWait = interval, busy, timeout, wait
	})
	st := &failingStorage{Storage: storage.NewMemStorage()}
	db, err := Open(st, nil)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { db.Close() })
	return db, st
}

func expectValue(t *testing.T, db *DB, key, want string) {
	t.Helper()
	got, err := db.Get([]byte(key), nil)
	if err != nil || string(got) != want {
		t.Fatalf("get %s = %q, %v; want %q", key, got, err, want)
	}
}

func TestWriteRecoversOnceStorageRecovers(t *testing.T) {
	db, st := openFailing(t)
	if err := db.Put([]byte("a"), []byte("1"), nil); err != nil {
		t.Fatal(err)
	}

	st.fail.Store(true)
	if err := db.Put([]byte("b"), []byte("2"), nil); !errors.Is(err, syscall.ENOSPC) {
		t.Fatalf("put while storage fails: %v", err)
	}
	expectValue(t, db, "a", "1")

	st.fail.Store(false)
	if err := db.Put([]byte("c"), []byte("3"), nil); err != nil {
		t.Fatalf("put after storage recovered: %v", err)
	}
	batch := new(leveldb.Batch)
	batch.Delete([]byte("c"))
	if err := db.Write(batch, nil); err != nil {
		t.Fatalf("batch after storage recovered: %v", err)
	}
	expectValue(t, db, "a", "1")
	if _, err := db.Get([]byte("c"), nil); err != leveldb.ErrNotFound {
		t.Fatalf("get deleted key: %v", err)
	}
}

func TestStalledProbeKeepsReadsAvailable(t *testing.T) {
	db, st := openFailing(t)
	if err := db.Put([]byte("a"), []byte("1"), nil); err != nil {
		t.Fatal(err)
	}
	st.probeStarted, st.probeRelease = make(chan struct{}, 1), make(chan struct{})
	st.fail.Store(true)
	put := make(chan error, 1)
	go func() { put <- db.Put([]byte("b"), []byte("2"), nil) }()
	<-st.probeStarted

	read := make(chan struct{})
	go func() {
		defer close(read)
		expectValue(t, db, "a", "1")
		db.NewIterator(nil, nil).Release()
	}()
	select {
	case <-read:
	case <-time.After(5 * time.Second):
		t.Error("reads blocked behind a stalled probe")
	}
	close(st.probeRelease)
	if err := <-put; !errors.Is(err, syscall.ENOSPC) {
		t.Fatalf("put while storage fails: %v", err)
	}
	<-read
}

func TestWritesDoNotQueueBehindAStalledProbe(t *testing.T) {
	db, st := openFailing(t)
	if err := db.Put([]byte("a"), []byte("1"), nil); err != nil {
		t.Fatal(err)
	}
	st.probeStarted, st.probeRelease = make(chan struct{}, 1), make(chan struct{})
	st.fail.Store(true)
	first := make(chan error, 1)
	go func() { first <- db.Put([]byte("b"), []byte("2"), nil) }()
	<-st.probeStarted

	second := make(chan error, 1)
	go func() { second <- db.Put([]byte("c"), []byte("3"), nil) }()
	select {
	case err := <-second:
		if !errors.Is(err, syscall.ENOSPC) {
			t.Errorf("put behind a stalled probe: %v", err)
		}
	case <-time.After(5 * time.Second):
		t.Error("put queued behind a stalled probe")
	}
	close(st.probeRelease)
	if err := <-first; !errors.Is(err, syscall.ENOSPC) {
		t.Fatalf("put while storage fails: %v", err)
	}
}

func TestReopenWaitsForOpenIterator(t *testing.T) {
	db, st := openFailing(t)
	for _, k := range []string{"a", "b"} {
		if err := db.Put([]byte(k), []byte(k), nil); err != nil {
			t.Fatal(err)
		}
	}
	st.fail.Store(true)
	if err := db.Put([]byte("x"), []byte("x"), nil); err == nil {
		t.Fatal("put while storage fails succeeded")
	}
	st.fail.Store(false)
	busyReopenInterval = time.Hour

	iter := db.NewIterator(nil, nil)
	if !iter.Next() || string(iter.Key()) != "a" {
		t.Fatalf("first key %q", iter.Key())
	}
	if err := db.Put([]byte("y"), []byte("y"), nil); !errors.Is(err, syscall.ENOSPC) {
		t.Fatalf("put with an open iterator: %v", err)
	}
	if !iter.Next() || string(iter.Key()) != "b" || iter.Next() {
		t.Fatalf("iterator after a refused reopen: key %q, err %v", iter.Key(), iter.Error())
	}
	iter.Release()
	iter.Release()

	if err := db.Put([]byte("y"), []byte("y"), nil); !errors.Is(err, syscall.ENOSPC) {
		t.Fatalf("put right after a refused reopen: %v", err)
	}
	db.nextReopen.Store(0)
	if err := db.Put([]byte("y"), []byte("y"), nil); err != nil {
		t.Fatalf("put after the iterator was released: %v", err)
	}
	expectValue(t, db, "b", "b")
}

func TestNestedReadWaitsOutTheDrain(t *testing.T) {
	db, st := openFailing(t)
	drainTimeout = 500 * time.Millisecond
	if err := db.Put([]byte("a"), []byte("1"), nil); err != nil {
		t.Fatal(err)
	}
	st.fail.Store(true)
	if err := db.Put([]byte("x"), []byte("x"), nil); err == nil {
		t.Fatal("put while storage fails succeeded")
	}
	st.fail.Store(false)
	st.probeStarted, st.probeRelease = make(chan struct{}, 1), make(chan struct{})

	iter := db.NewIterator(nil, nil)
	put := make(chan error, 1)
	go func() { put <- db.Put([]byte("y"), []byte("y"), nil) }()
	<-st.probeStarted
	close(st.probeRelease)
	for db.draining.Load() == nil {
		select {
		case err := <-put:
			t.Fatalf("put returned before its drain was seen: %v", err)
		case <-time.After(time.Millisecond):
		}
	}
	expectValue(t, db, "a", "1")
	iter.Release()
	if err := <-put; !errors.Is(err, syscall.ENOSPC) {
		t.Fatalf("put with an open iterator: %v", err)
	}
}

func TestReopenIsNotStarvedByOverlappingReaders(t *testing.T) {
	db, st := openFailing(t)
	drainTimeout = 5 * time.Second
	if err := db.Put([]byte("a"), []byte("1"), nil); err != nil {
		t.Fatal(err)
	}
	st.fail.Store(true)
	if err := db.Put([]byte("x"), []byte("x"), nil); err == nil {
		t.Fatal("put while storage fails succeeded")
	}
	st.fail.Store(false)

	stop := make(chan struct{})
	var readers, started sync.WaitGroup
	for i := 0; i < 4; i++ {
		readers.Add(1)
		started.Add(1)
		go func() {
			defer readers.Done()
			for first := true; ; first = false {
				select {
				case <-stop:
					return
				default:
				}
				iter := db.NewIterator(nil, nil)
				if first {
					started.Done()
				}
				time.Sleep(2 * time.Millisecond)
				iter.Release()
			}
		}()
		time.Sleep(500 * time.Microsecond)
	}
	started.Wait()
	err := db.Put([]byte("y"), []byte("y"), nil)
	close(stop)
	readers.Wait()
	if err != nil {
		t.Fatalf("put while readers overlap: %v", err)
	}
	expectValue(t, db, "y", "y")
}

func TestCloseIsNotUndoneByReopen(t *testing.T) {
	db, st := openFailing(t)
	st.fail.Store(true)
	if err := db.Put([]byte("a"), []byte("1"), nil); err == nil {
		t.Fatal("put while storage fails succeeded")
	}
	st.fail.Store(false)
	db.Close()
	if err := db.Put([]byte("a"), []byte("1"), nil); err != leveldb.ErrClosed {
		t.Fatalf("put after close: %v", err)
	}
	if iter := db.NewIterator(nil, nil); iter.Next() || iter.Error() != leveldb.ErrClosed {
		t.Fatalf("iterator after close: %v", iter.Error())
	}
}

func TestReopenKeepsFileDatabase(t *testing.T) {
	dir := t.TempDir()
	db, err := OpenFile(dir, nil)
	if err != nil {
		t.Fatal(err)
	}
	if err := db.Put([]byte("a"), []byte("1"), nil); err != nil {
		t.Fatal(err)
	}
	if !db.reopen(db.db) {
		t.Fatal("reopen failed")
	}
	if err := db.Put([]byte("b"), []byte("2"), nil); err != nil {
		t.Fatal(err)
	}
	if err := db.Close(); err != nil {
		t.Fatal(err)
	}

	db, err = OpenFile(dir, nil)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()
	expectValue(t, db, "a", "1")
	expectValue(t, db, "b", "2")
}

func TestReopenAttemptsAreRateLimited(t *testing.T) {
	db, st := openFailing(t)
	reopenInterval = time.Hour
	st.fail.Store(true)
	if err := db.Put([]byte("a"), []byte("1"), nil); err == nil {
		t.Fatal("put while storage fails succeeded")
	}
	st.fail.Store(false)
	if err := db.Put([]byte("a"), []byte("1"), nil); !errors.Is(err, syscall.ENOSPC) {
		t.Fatalf("put within the reopen interval: %v", err)
	}
	db.nextReopen.Store(0)
	if err := db.Put([]byte("a"), []byte("1"), nil); err != nil {
		t.Fatalf("put after the reopen interval: %v", err)
	}
}

func TestFailedReopenReportsErrorsUntilOpen(t *testing.T) {
	db, st := openFailing(t)
	if err := db.Put([]byte("a"), []byte("1"), nil); err != nil {
		t.Fatal(err)
	}
	st.fail.Store(true)
	if err := db.Put([]byte("b"), []byte("2"), nil); err == nil {
		t.Fatal("put while storage fails succeeded")
	}
	st.fail.Store(false)
	st.failOpen.Store(true)
	if err := db.Put([]byte("b"), []byte("2"), nil); err == nil {
		t.Fatal("put with a failed reopen succeeded")
	}
	if _, err := db.Get([]byte("a"), nil); !errors.Is(err, syscall.ENOSPC) {
		t.Fatalf("get with a failed reopen: %v", err)
	}
	iter := db.NewIterator(nil, nil)
	if iter.Next() || !errors.Is(iter.Error(), syscall.ENOSPC) {
		t.Fatalf("iterator with a failed reopen: %v", iter.Error())
	}
	iter.Release()

	st.failOpen.Store(false)
	expectValue(t, db, "a", "1")
	if err := db.Put([]byte("b"), []byte("2"), nil); err != nil {
		t.Fatalf("put after the reopen succeeded: %v", err)
	}
}