mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-23 14:30:46 +02:00
Jaehoon KimandChris Lu
744ab586c5
* shell: volume.fsck reads the filer from one store snapshot when it can volume.fsck collects the filer's chunk references by walking directory by directory, so each directory is seen at a different time. A file renamed from a directory not yet listed into one already listed is never seen, its chunks look like orphans, and -reallyDeleteFromVolume deletes live data. Checkpoint rotation and write-to-tmp-then-rename hit this. Add an optional filer store capability, EntryScanner, that visits every entry below a directory from one consistent snapshot. The SQL stores implement it with one REPEATABLE READ read-only transaction read in primary-key pages, so a concurrent rename is seen under exactly one of its names. Key-value rows are skipped by their dirhash, a bucket directory whose table is missing fails the scan (from the snapshot it cannot be told from one dropped a moment ago), and hard links are resolved in the same transaction. SQLite offers it only in WAL mode, on its own connection. TraverseBfsMetadata serves the snapshot when asked via request metadata sw-traverse-mode=snapshot and reports the mode in a response header; without the metadata it walks as before, so peer bootstrap and admin export are unchanged. Snapshot responses carry no entry_ts_ns, whose per-entry lookup would cost more than the scan. volume.fsck requests the snapshot and falls back to the walk on an older filer or a store that cannot scan. * filer, shell: fail a scan that cannot start, not fall back to the walk A transient BeginTx failure and a preliminary GetEntry error were treated like a missing capability or missing root, so fsck would take the directory walk - the very path that misses renamed-away entries. Only a genuinely unsupported scan or a confirmed-absent root may fall back. * filer: only snapshot-listed bucket names keep a missing table fatal A cached-only name, whose bucket may have been deleted between copying the cache and opening the snapshot, is skipped when its table is gone. * filer: roll back to a savepoint before skipping a cached-only table On PostgreSQL a failed statement aborts the snapshot transaction, so ignoring a missing-table error without the savepoint still left every later cached-only table unreadable. --------- Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
72 lines
2.9 KiB
Go
72 lines
2.9 KiB
Go
package filer
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
const CountEntryChunksForGzip = 50
|
|
|
|
var (
|
|
ErrUnsupportedListDirectoryPrefixed = errors.New("unsupported directory prefix listing")
|
|
ErrUnsupportedSuperLargeDirectoryListing = errors.New("unsupported super large directory listing")
|
|
ErrKvNotImplemented = errors.New("kv not implemented yet")
|
|
ErrKvNotFound = errors.New("kv: not found")
|
|
)
|
|
|
|
type ListEachEntryFunc func(entry *Entry) (bool, error)
|
|
|
|
type FilerStore interface {
|
|
// GetName gets the name to locate the configuration in filer.toml file
|
|
GetName() string
|
|
// Initialize initializes the file store
|
|
Initialize(configuration util.Configuration, prefix string) error
|
|
InsertEntry(context.Context, *Entry) error
|
|
UpdateEntry(context.Context, *Entry) (err error)
|
|
// err == filer_pb.ErrNotFound if not found
|
|
FindEntry(context.Context, util.FullPath) (entry *Entry, err error)
|
|
DeleteEntry(context.Context, util.FullPath) (err error)
|
|
DeleteFolderChildren(context.Context, util.FullPath) (err error)
|
|
ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)
|
|
ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)
|
|
|
|
BeginTransaction(ctx context.Context) (context.Context, error)
|
|
CommitTransaction(ctx context.Context) error
|
|
RollbackTransaction(ctx context.Context) error
|
|
|
|
KvPut(ctx context.Context, key []byte, value []byte) (err error)
|
|
KvGet(ctx context.Context, key []byte) (value []byte, err error)
|
|
KvDelete(ctx context.Context, key []byte) (err error)
|
|
|
|
Shutdown()
|
|
}
|
|
|
|
// ErrUnsupportedEntryScan is returned by EntryScanner.ScanEntries, before
|
|
// any entry is visited, when the store cannot serve the scan.
|
|
var ErrUnsupportedEntryScan = errors.New("entry scan not supported by this filer store")
|
|
|
|
// EntryScanner is an optional FilerStore capability: visit every entry at and
|
|
// below dirPath, in any order, from one consistent snapshot of the store.
|
|
//
|
|
// A directory walk sees each directory at a different time, so an entry
|
|
// renamed from a directory not yet listed into one already listed is never
|
|
// seen; volume.fsck then purges its chunks as orphans. A snapshot sees it
|
|
// exactly once. The context handed to fn carries the snapshot, so store reads
|
|
// made with it (hard-link records) see the same instant.
|
|
type EntryScanner interface {
|
|
ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *Entry) error) error
|
|
}
|
|
|
|
type BucketAware interface {
|
|
OnBucketCreation(bucket string)
|
|
OnBucketDeletion(bucket string)
|
|
CanDropWholeBucket() bool
|
|
}
|
|
|
|
type Debuggable interface {
|
|
Debug(writer io.Writer)
|
|
}
|