filer: recover a leveldb store whose table a power loss damaged (#12780)

* filer: recover leveldb without rebuilding broken tables

leveldb.Recover rebuilds a table with damaged blocks using the user
comparer and filter on internal keys. The shortened index keys it writes
are not valid internal keys, so a Get on the rebuilt table panics, and
the bloom filter it writes misses every lookup. Recover with a comparer
that keeps full index keys (same name, so the manifest still matches)
and no filter.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: recover a leveldb store when a compaction finds a corrupted table

When a compaction reads a damaged table block, as a power loss can
leave one, goleveldb latches the corruption error and returns it for
every later write until the database is reopened. A plain reopen loads
the same table and the next compaction fails again, so the store kept
rejecting inserts even across restarts.

Treat a corruption error from a write like a storage error, and reopen
through leveldb.Recover, which rebuilds the table without the damaged
blocks, then retry the write.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: rebuild damaged leveldb tables before Recover

leveldb.Recover's own table rebuild orders internal keys with the user
comparer, so a damaged table holding several versions of one key fails
with "keys are not in increasing order" and the recovery gives up.
Overriding the comparer cannot fix that, because the same comparer
also serves the recovered database's user keys.

Rebuild every table that has corrupted blocks first, using an internal
key comparer and full index keys, then let Recover rebuild the manifest
from tables it no longer needs to rewrite. Seed the test tables from a
journal flush so they keep several versions of a key, and check keys
the write test does not rewrite.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: leave unreadable leveldb tables for Recover to drop

table.NewReader reports a truncated table or a damaged footer through
the reader, not its error, so rebuildDamagedTables never stopped on one,
but it rewrote it as an empty table first. Skip tables with nothing
readable and let Recover drop them, and test an unreadable table next
to a healthy one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: let StrictRecovery drop damaged leveldb tables

With StrictRecovery set, leveldb.Recover drops a table that has corrupted
blocks. Salvaging the readable blocks first would hide the damage from it,
so skip the rebuild under that option.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: hold the leveldb storage lock while rebuilding tables

The table rebuild ran after the failed open released the storage lock
and before Recover took it, so it could rewrite the tables of a database
another caller had open on the same storage. Take the lock for the
rebuild; Recover takes it again afterwards.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed 2026-11-19 13:13:30 -08:00
1 parent 6bdcd17ed4
commit c9f3a02b94
3 files changed
+414 -16

No files matched your search

+22 -15
View File
@@ -1,7 +1,9 @@
// Package leveldb_handle wraps a goleveldb database so a write that fails on
// a storage error reopens it. goleveldb latches the first journal write error
// and returns it for every later write until the database is reopened, so one
// transient ENOSPC or EIO would otherwise fail writes until a restart.
// a storage error or a corrupted table reopens it. goleveldb latches the first
// journal write error, and the first corruption a compaction reads, and
// returns it for every later write until the database is reopened. Corruption
// reopens through leveldb.Recover, which drops the damaged blocks; a plain
// open would load the same table and fail again.
package leveldb_handle
import (
@@ -65,19 +67,24 @@ func OpenFile(path string, o *opt.Options) (*DB, error) {
}
func Open(stor storage.Storage, o *opt.Options) (*DB, error) {
db, err := open(stor, o)
db, err := open(stor, o, nil)
if err != nil {
return nil, err
}
return &DB{stor: stor, opts: o, db: db, reopening: make(chan struct{}, 1)}, nil
}
func open(stor storage.Storage, o *opt.Options) (*leveldb.DB, error) {
db, err := leveldb.Open(stor, o)
if leveldb_errors.IsCorrupted(err) {
db, err = leveldb.Recover(stor, o)
func open(stor storage.Storage, o *opt.Options, cause error) (*leveldb.DB, error) {
if !leveldb_errors.IsCorrupted(cause) {
db, err := leveldb.Open(stor, o)
if !leveldb_errors.IsCorrupted(err) {
return db, err
}
}
return db, err
if err := rebuildDamagedTables(stor, o); err != nil {
return nil, err
}
return leveldb.Recover(stor, o)
}
func (d *DB) Get(key []byte, ro *opt.ReadOptions) ([]byte, error) {
@@ -136,7 +143,7 @@ func (d *DB) write(fn func(*leveldb.DB) error) error {
}
err = fn(db)
d.mu.RUnlock()
if !isStorageError(err) || !d.reopen(db) {
if !isStorageError(err) || !d.reopen(db, err) {
return err
}
if db, err = d.acquire(); err != nil {
@@ -162,7 +169,7 @@ func (d *DB) acquire() (*leveldb.DB, error) {
}
err := d.openErr
d.mu.RUnlock()
if attempt > 0 || !d.reopen(nil) {
if attempt > 0 || !d.reopen(nil, err) {
return nil, err
}
}
@@ -171,7 +178,7 @@ func (d *DB) acquire() (*leveldb.DB, error) {
// reopen replaces failed with a freshly opened database once no call holds
// it, unless another caller already did. It waits at most reopenWait for a
// running reopen and reports whether a retry can use an open database.
func (d *DB) reopen(failed *leveldb.DB) bool {
func (d *DB) reopen(failed *leveldb.DB, cause error) bool {
select {
case d.reopening <- struct{}{}:
case <-time.After(reopenWait):
@@ -202,11 +209,11 @@ func (d *DB) reopen(failed *leveldb.DB) bool {
if d.closed.Load() {
return false
}
glog.Warningf("leveldb %s: reopening after a storage write error", d.name)
glog.Warningf("leveldb %s: reopening after %v", d.name, cause)
if failed != nil {
failed.Close()
}
d.db, d.openErr = open(d.stor, d.opts)
d.db, d.openErr = open(d.stor, d.opts, cause)
if d.openErr != nil {
glog.Errorf("leveldb %s: reopen: %v", d.name, d.openErr)
return false
@@ -255,7 +262,7 @@ func probe(stor storage.Storage) error {
func isStorageError(err error) bool {
var errno syscall.Errno
return errors.As(err, &errno)
return errors.As(err, &errno) || leveldb_errors.IsCorrupted(err)
}
type heldIterator struct {
@@ -2,6 +2,11 @@ package leveldb_handle
import (
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
"syscall"
@@ -9,6 +14,9 @@ import (
"time"
"github.com/syndtr/goleveldb/leveldb"
leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/filter"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/storage"
)
@@ -293,7 +301,7 @@ func TestReopenKeepsFileDatabase(t *testing.T) {
if err := db.Put([]byte("a"), []byte("1"), nil); err != nil {
t.Fatal(err)
}
if !db.reopen(db.db) {
if !db.reopen(db.db, nil) {
t.Fatal("reopen failed")
}
if err := db.Put([]byte("b"), []byte("2"), nil); err != nil {
@@ -358,3 +366,229 @@ func TestFailedReopenReportsErrorsUntilOpen(t *testing.T) {
t.Fatalf("put after the reopen succeeded: %v", err)
}
}
// damagedTableDB writes keys k0000..k1999, k1000 in several versions, into
// tables and zeroes part of the first data block of each table, as a power
// loss can leave it.
func damagedTableDB(t *testing.T) (dir string, opts *opt.Options, value string) {
t.Helper()
dir = t.TempDir()
opts = &opt.Options{Filter: filter.NewBloomFilter(8)}
value = strings.Repeat("v", 100)
raw, err := leveldb.OpenFile(dir, opts)
if err != nil {
t.Fatal(err)
}
for _, v := range []string{"old1", "old2"} {
if err := raw.Put([]byte("k1000"), []byte(v), nil); err != nil {
t.Fatal(err)
}
}
for i := 0; i < 2000; i++ {
if err := raw.Put([]byte(fmt.Sprintf("k%04d", i)), []byte(value), nil); err != nil {
t.Fatal(err)
}
}
// Reopening flushes the journal into one level-0 table that keeps every
// version of k1000.
for reopen := 0; reopen < 2; reopen++ {
if err := raw.Close(); err != nil {
t.Fatal(err)
}
if raw, err = leveldb.OpenFile(dir, opts); err != nil {
t.Fatal(err)
}
}
if err := raw.Close(); err != nil {
t.Fatal(err)
}
tables, _ := filepath.Glob(filepath.Join(dir, "*.ldb"))
if len(tables) == 0 {
t.Fatal("no table files")
}
for _, table := range tables {
f, err := os.OpenFile(table, os.O_RDWR, 0)
if err != nil {
t.Fatal(err)
}
_, err = f.WriteAt(make([]byte, 64), 64)
f.Close()
if err != nil {
t.Fatal(err)
}
}
return dir, opts, value
}
func TestOpenRecoversDamagedTable(t *testing.T) {
dir, opts, value := damagedTableDB(t)
if err := os.Remove(filepath.Join(dir, "CURRENT")); err != nil {
t.Fatal(err)
}
db, err := OpenFile(dir, opts)
if err != nil {
t.Fatal(err)
}
expectValue(t, db, "k1000", value)
expectValue(t, db, "k1999", value)
if err := db.Put([]byte("after"), []byte("1"), nil); err != nil {
t.Fatal(err)
}
if err := db.Close(); err != nil {
t.Fatal(err)
}
db, err = OpenFile(dir, opts)
if err != nil {
t.Fatal(err)
}
defer db.Close()
expectValue(t, db, "k1999", value)
expectValue(t, db, "after", "1")
}
func TestWriteRecoversFromCorruptedTable(t *testing.T) {
dir, opts, value := damagedTableDB(t)
opts.WriteBuffer = 64 * 1024
opts.CompactionTotalSize = 64 * 1024
db, err := OpenFile(dir, opts)
if err != nil {
t.Fatal(err)
}
if _, err := db.Get([]byte("k0000"), nil); !leveldb_errors.IsCorrupted(err) {
t.Fatalf("get from the damaged block: %v", err)
}
db.mu.RLock()
damaged := db.db
db.mu.RUnlock()
// Overwrite the keys until a background compaction reads the damaged
// table and goleveldb fails every later write.
for round := 0; ; round++ {
db.mu.RLock()
recovered := db.db != damaged
db.mu.RUnlock()
if recovered {
break
}
if round == 100 {
t.Fatal("no compaction reached the damaged table")
}
for i := 0; i < 1000; i++ {
if err := db.Put([]byte(fmt.Sprintf("k%04d", i)), []byte("new"), nil); err != nil {
t.Fatalf("put after a compaction hit the damaged table: %v", err)
}
}
}
if err := db.Put([]byte("after"), []byte("1"), nil); err != nil {
t.Fatal(err)
}
expectValue(t, db, "k0999", "new")
expectValue(t, db, "k1000", value)
expectValue(t, db, "k1999", value)
if err := db.Close(); err != nil {
t.Fatal(err)
}
db, err = OpenFile(dir, opts)
if err != nil {
t.Fatal(err)
}
defer db.Close()
expectValue(t, db, "after", "1")
expectValue(t, db, "k0000", "new")
expectValue(t, db, "k1000", value)
expectValue(t, db, "k1999", value)
}
func TestOpenDropsUnreadableTable(t *testing.T) {
dir := t.TempDir()
opts := &opt.Options{Filter: filter.NewBloomFilter(8)}
for _, key := range []string{"a", "b", "c"} {
raw, err := leveldb.OpenFile(dir, opts)
if err != nil {
t.Fatal(err)
}
if err := raw.Put([]byte(key), []byte(key), nil); err != nil {
t.Fatal(err)
}
if err := raw.Close(); err != nil {
t.Fatal(err)
}
}
// Each reopen flushed the previous journal, so "a" and "b" have a table
// each and "c" is still in the journal.
tables, _ := filepath.Glob(filepath.Join(dir, "*.ldb"))
if len(tables) != 2 {
t.Fatalf("tables: %v", tables)
}
sort.Strings(tables)
if err := os.Truncate(tables[0], 10); err != nil {
t.Fatal(err)
}
if err := os.Remove(filepath.Join(dir, "CURRENT")); err != nil {
t.Fatal(err)
}
db, err := OpenFile(dir, opts)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if _, err := db.Get([]byte("a"), nil); err != leveldb.ErrNotFound {
t.Fatalf("get from the unreadable table: %v", err)
}
expectValue(t, db, "b", "b")
expectValue(t, db, "c", "c")
if err := db.Put([]byte("d"), []byte("d"), nil); err != nil {
t.Fatal(err)
}
}
func TestStrictRecoveryDropsDamagedTable(t *testing.T) {
dir, opts, _ := damagedTableDB(t)
if err := os.Remove(filepath.Join(dir, "CURRENT")); err != nil {
t.Fatal(err)
}
opts.Strict = opt.DefaultStrict | opt.StrictRecovery
db, err := OpenFile(dir, opts)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if _, err := db.Get([]byte("k1999"), nil); err != leveldb.ErrNotFound {
t.Fatalf("get from the dropped table: %v", err)
}
}
func TestRebuildLeavesTablesOfAnOpenDatabase(t *testing.T) {
dir, opts, _ := damagedTableDB(t)
stor, err := storage.OpenFile(dir, false)
if err != nil {
t.Fatal(err)
}
defer stor.Close()
raw, err := leveldb.Open(stor, opts)
if err != nil {
t.Fatal(err)
}
defer raw.Close()
tables, _ := filepath.Glob(filepath.Join(dir, "*.ldb"))
before, err := os.ReadFile(tables[0])
if err != nil {
t.Fatal(err)
}
if err := rebuildDamagedTables(stor, opts); err != storage.ErrLocked {
t.Fatalf("rebuild under an open database: %v", err)
}
after, err := os.ReadFile(tables[0])
if err != nil {
t.Fatal(err)
}
if string(after) != string(before) {
t.Fatal("table rewritten under an open database")
}
}
+157
View File
@@ -0,0 +1,157 @@
package leveldb_handle
import (
"encoding/binary"
"io"
"github.com/syndtr/goleveldb/leveldb/comparer"
leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/syndtr/goleveldb/leveldb/storage"
"github.com/syndtr/goleveldb/leveldb/table"
)
const internalKeyTrailer = 8
// rebuildDamagedTables rewrites every table that has corrupted blocks
// without them, before leveldb.Recover runs. Recover would rebuild such a
// table itself, but it orders internal keys with the user comparer, so a
// table holding several versions of one key fails to rebuild, and the index
// keys and filter it writes do not match lookups. A table with nothing
// readable is left for Recover to drop, and so is every damaged table under
// StrictRecovery.
func rebuildDamagedTables(stor storage.Storage, o *opt.Options) error {
if o.GetStrict(opt.StrictRecovery) {
return nil
}
lock, err := stor.Lock()
if err != nil {
return err
}
defer lock.Unlock()
fds, err := stor.List(storage.TypeTable)
if err != nil {
return err
}
to := &opt.Options{
Comparer: internalComparer{o.GetComparer()},
Compression: o.GetCompression(),
BlockSize: o.GetBlockSize(),
Strict: opt.DefaultStrict &^ opt.StrictReader,
}
for _, fd := range fds {
if err := rebuildTable(stor, fd, to); err != nil {
return err
}
}
return nil
}
func rebuildTable(stor storage.Storage, fd storage.FileDesc, o *opt.Options) error {
r, err := stor.Open(fd)
if err != nil {
return err
}
readerOpen := true
defer func() {
if readerOpen {
r.Close()
}
}()
size, err := r.Seek(0, io.SeekEnd)
if err != nil {
return err
}
tr, err := table.NewReader(r, size, fd, nil, nil, o)
if err != nil {
return err
}
entries, damaged, err := scanTable(tr, nil)
if err != nil || !damaged || entries == 0 {
return err
}
tmp := storage.FileDesc{Type: storage.TypeTemp, Num: fd.Num}
w, err := stor.Create(tmp)
if err != nil {
return err
}
tw := table.NewWriter(w, o)
if _, _, err = scanTable(tr, tw); err == nil {
err = tw.Close()
}
if err == nil {
err = w.Sync()
}
if closeErr := w.Close(); err == nil {
err = closeErr
}
if err == nil {
readerOpen = false
r.Close()
err = stor.Rename(tmp, fd)
}
if err != nil {
stor.Remove(tmp)
}
return err
}
// scanTable counts the readable entries of tr, appending them to tw if set,
// and reports whether a block was corrupted.
func scanTable(tr *table.Reader, tw *table.Writer) (entries int, damaged bool, err error) {
iter := tr.NewIterator(nil, nil)
defer iter.Release()
if setter, ok := iter.(iterator.ErrorCallbackSetter); ok {
setter.SetErrorCallback(func(err error) {
if leveldb_errors.IsCorrupted(err) {
damaged = true
}
})
}
for iter.Next() {
if len(iter.Key()) < internalKeyTrailer {
continue
}
entries++
if tw != nil {
if err := tw.Append(iter.Key(), iter.Value()); err != nil {
return entries, damaged, err
}
}
}
if err := iter.Error(); err != nil {
if !leveldb_errors.IsCorrupted(err) {
return entries, damaged, err
}
damaged = true
}
return entries, damaged, nil
}
// internalComparer orders goleveldb internal keys, a user key followed by
// a little-endian sequence and kind, newest version first. It keeps full
// index keys instead of shortening them.
type internalComparer struct {
comparer.Comparer
}
func (c internalComparer) Compare(a, b []byte) int {
if r := c.Comparer.Compare(a[:len(a)-internalKeyTrailer], b[:len(b)-internalKeyTrailer]); r != 0 {
return r
}
m := binary.LittleEndian.Uint64(a[len(a)-internalKeyTrailer:])
n := binary.LittleEndian.Uint64(b[len(b)-internalKeyTrailer:])
switch {
case m > n:
return -1
case m < n:
return 1
}
return 0
}
func (internalComparer) Separator(dst, a, b []byte) []byte { return nil }
func (internalComparer) Successor(dst, b []byte) []byte { return nil }