Merge branch 'master' of https://github.com/seaweedfs/seaweedfs into nfs-enable-inode-index

This commit is contained in:
Chris Lu committed 2026-10-23 01:16:00 -07:00
commit 347d02cf9e
213 files changed
+16701 -1571

No files matched your search

+11 -2
View File
@@ -23,14 +23,20 @@ type SqlGenerator interface {
GetSqlDeleteFolderChildren(tableName string) string
GetSqlListExclusive(tableName string) string
GetSqlListInclusive(tableName string) string
// GetSqlScanPage returns the query and arguments selecting dirhash, name,
// directory and meta of up to limit rows after the (dirhash, name) cursor.
GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{})
GetSqlCreateTable(tableName string) string
GetSqlDropTable(tableName string) string
}
type AbstractSqlStore struct {
SqlGenerator
DB *sql.DB
KvDB *sql.DB
DB *sql.DB
KvDB *sql.DB
// ScanDB, when set, is the pool snapshot scans run on (SQLite's main pool
// is a single connection that a long scan must not hold).
ScanDB *sql.DB
SupportBucketTable bool
SkipDDL bool
dbs map[string]bool
@@ -457,6 +463,9 @@ func (store *AbstractSqlStore) Shutdown() {
if store.KvDB != nil {
store.KvDB.Close()
}
if store.ScanDB != nil {
store.ScanDB.Close()
}
}
func isValidBucket(bucket string) bool {
@@ -0,0 +1,368 @@
package abstract_sql
import (
"context"
"database/sql"
"fmt"
"math"
"strings"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var _ filer.EntryScanner = (*AbstractSqlStore)(nil)
// scanTable is one table to read during a scan together with the path its
// directory column is relative to: "" for the default table, whose rows hold
// full directories, and "/buckets/<bucket>" for a per-bucket table.
type scanTable struct {
table string
base string
// cachedOnly marks a table that only a stale bucket cache remembers: its
// bucket directory was already gone from the snapshot, so a missing table
// means it was dropped meanwhile, not that rows are absent.
cachedOnly bool
}
// scanPageSize is how many rows one keyset page of a scan fetches.
var scanPageSize = 1000
// SetScanPageSizeForTest lets tests drive the keyset cursor across pages. It
// returns the previous size.
func SetScanPageSizeForTest(n int) int {
prev := scanPageSize
scanPageSize = n
return prev
}
// ScanEntries implements filer.EntryScanner: every table that can hold an
// entry below dirPath is read inside one REPEATABLE READ, read-only
// transaction, in primary-key pages. All rows are seen as of one instant; a
// rename committed meanwhile shows up under exactly one of its names. The
// transaction takes no row locks and does not block writers; it does hold the
// tables it has read against DDL, so a bucket deleted while the scan runs
// drops its table once the scan ends. A page's cursor is closed before its
// entries go to fn, so fn may read the store through the context it receives
// (which carries the transaction) on the same connection.
func (store *AbstractSqlStore) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error {
// Read before the transaction holds a connection: OnBucketCreation keeps
// dbsLock while it waits for a connection of its own, so taking the lock
// while holding a connection could deadlock on an exhausted pool.
cachedBuckets := store.cachedBucketNames()
tx, err := store.beginSnapshotTx(ctx)
if err != nil {
return err
}
// Read-only: nothing to commit, and a rollback releases the snapshot.
defer tx.Rollback()
tables, err := store.scanTables(ctx, tx, dirPath, cachedBuckets)
if err != nil {
return err
}
// Reads made through this context (getTxOrDB, kvDB) use the transaction
// and therefore the snapshot.
txCtx := context.WithValue(ctx, "tx", tx)
// The root's own row may live in a table the scan does not read (a
// bucket directory is in the default table, its contents in the bucket's).
if root, err := store.readRowInSnapshot(txCtx, tx, dirPath); err != nil {
return err
} else if root != nil {
if err := fn(txCtx, root); err != nil {
return err
}
}
for _, t := range tables {
if err := store.scanOneTable(txCtx, tx, t, dirPath, fn); err != nil {
return err
}
}
return nil
}
// readRowInSnapshot reads one entry's row inside the transaction, routing to
// the table getTxOrDB would use but without its bucket-table creation. A
// missing row is nil.
func (store *AbstractSqlStore) readRowInSnapshot(ctx context.Context, tx *sql.Tx, p util.FullPath) (*filer.Entry, error) {
if p == "/" {
return nil, nil
}
parent, name := p.DirAndName()
table, shortDir := DEFAULT_TABLE, parent
if store.SupportBucketTable {
if bucket := bucketTableFor(util.FullPath(parent)); bucket != "" {
table = bucket
if shortDir = strings.TrimPrefix(parent, bucketBase(bucket)); shortDir == "" {
shortDir = "/"
}
}
}
var data []byte
err := tx.QueryRowContext(ctx, store.GetSqlFind(table), util.HashStringToLong(shortDir), name, shortDir).Scan(&data)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, missingTableOr(err, table, fmt.Sprintf("read %s", p))
}
entry := &filer.Entry{FullPath: p}
if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
return nil, fmt.Errorf("decode %s: %v", p, err)
}
return entry, nil
}
// beginSnapshotTx starts the read-only REPEATABLE READ transaction that gives
// one snapshot across statements. Nothing weaker is tried: under READ
// COMMITTED each page would see a different snapshot, and the stream would
// still be advertised as one.
func (store *AbstractSqlStore) beginSnapshotTx(ctx context.Context) (*sql.Tx, error) {
db := store.DB
if store.ScanDB != nil {
db = store.ScanDB
} else if db.Stats().MaxOpenConnections == 1 {
// The scan would hold the filer's only connection for its whole run.
return nil, fmt.Errorf("%w: the store's single connection cannot be held for a scan", filer.ErrUnsupportedEntryScan)
}
// A BeginTx failure (a down database) must fail the traversal, not route it
// to the walk: a recovered database lets the walk read renamed-away entries
// as absent, the very loss this scan exists to prevent.
tx, err := db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
if err != nil {
return nil, fmt.Errorf("begin snapshot: %v", err)
}
return tx, nil
}
func (store *AbstractSqlStore) cachedBucketNames() []string {
store.dbsLock.Lock()
defer store.dbsLock.Unlock()
names := make([]string, 0, len(store.dbs))
for name := range store.dbs {
names = append(names, name)
}
return names
}
// scanTables picks the tables to read for dirPath: the default table alone,
// one bucket's table for a path inside that bucket, or the default table plus
// every bucket table (enumerated from /buckets inside the snapshot).
func (store *AbstractSqlStore) scanTables(ctx context.Context, tx *sql.Tx, dirPath util.FullPath, cachedBuckets []string) ([]scanTable, error) {
if !store.SupportBucketTable {
return []scanTable{{table: DEFAULT_TABLE}}, nil
}
if bucket := bucketTableFor(dirPath); bucket != "" {
return []scanTable{{table: bucket, base: bucketBase(bucket)}}, nil
}
tables := []scanTable{{table: DEFAULT_TABLE}}
if dirPath != "/" && string(dirPath) != bucketsRoot {
// Outside every bucket (e.g. /topics, or an internal /buckets/.system
// folder): everything below dirPath is in the default table.
return tables, nil
}
buckets, err := store.listBucketNames(ctx, tx, cachedBuckets)
if err != nil {
return nil, err
}
for _, bucket := range buckets.names {
tables = append(tables, scanTable{table: bucket, base: bucketBase(bucket), cachedOnly: !buckets.seen[bucket]})
}
return tables, nil
}
const bucketsRoot = "/buckets"
func bucketBase(bucket string) string {
return bucketsRoot + "/" + bucket
}
// bucketTableFor returns the bucket whose table alone holds everything below
// dirPath, or "" when dirPath is not inside a real bucket (as getTxOrDB routes).
func bucketTableFor(dirPath util.FullPath) string {
p := string(dirPath)
if !strings.HasPrefix(p, bucketsRoot+"/") {
return ""
}
rest := p[len(bucketsRoot)+1:]
bucket := rest
if i := strings.Index(rest, "/"); i >= 0 {
bucket = rest[:i]
}
if bucket == "" || strings.HasPrefix(bucket, ".") || !isValidBucket(bucket) {
return ""
}
return bucket
}
// bucketNames is the snapshot-listed and cached-only buckets whose tables a
// scan may read.
type bucketNames struct {
names []string
seen map[string]bool // listed under /buckets inside the snapshot
}
// listBucketNames reads the directories under /buckets within the snapshot
// and keeps those that route to their own table, plus the buckets this
// process has already seen (their directory entry may be gone, their rows not).
func (store *AbstractSqlStore) listBucketNames(ctx context.Context, tx *sql.Tx, cachedBuckets []string) (bucketNames, error) {
var names bucketNames
names.seen = make(map[string]bool)
added := make(map[string]bool)
add := func(name string) {
if added[name] || strings.HasPrefix(name, ".") || !isValidBucket(name) {
return
}
added[name] = true
names.names = append(names.names, name)
}
const page = 1000
startFrom := ""
for {
type row struct {
name string
data []byte
}
var rows []row
if err := func() error {
res, err := tx.QueryContext(ctx, store.GetSqlListExclusive(DEFAULT_TABLE), util.HashStringToLong(bucketsRoot), startFrom, bucketsRoot, "%", page)
if err != nil {
return fmt.Errorf("list %s: %v", bucketsRoot, err)
}
defer res.Close()
for res.Next() {
var r row
if err := res.Scan(&r.name, &r.data); err != nil {
return fmt.Errorf("scan %s: %v", bucketsRoot, err)
}
rows = append(rows, r)
}
return res.Err()
}(); err != nil {
return bucketNames{}, err
}
for _, r := range rows {
startFrom = r.name
entry := &filer.Entry{FullPath: util.NewFullPath(bucketsRoot, r.name)}
if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(r.data)); err != nil {
return bucketNames{}, fmt.Errorf("scan decode %s: %v", entry.FullPath, err)
}
if entry.IsDirectory() {
names.seen[r.name] = true
add(r.name)
}
}
if len(rows) < page {
break
}
}
for _, name := range cachedBuckets {
add(name)
}
return names, nil
}
// scanOneTable reads one table in primary-key pages and hands the entries
// strictly below dirPath to fn (the root itself is read separately). Key-value
// records share the default table; GenDirAndName stores the raw key bytes as
// their dirhash while entry rows hash the directory, so the mismatch
// identifies them exactly.
func (store *AbstractSqlStore) scanOneTable(ctx context.Context, tx *sql.Tx, t scanTable, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error {
type row struct {
dirHash int64
name, dir string
data []byte
}
cursorHash, cursorName := int64(math.MinInt64), ""
firstPage := true
for {
var page []row
if t.cachedOnly && firstPage {
// A missing table can only be met by the first page read: once a
// row came back the tx holds the table against a drop. On
// PostgreSQL the failed query aborts the transaction, so the skip
// needs a savepoint to roll back to before the next table's read.
if _, err := tx.ExecContext(ctx, "SAVEPOINT seaweed_scan_bucket"); err != nil {
return fmt.Errorf("scan table %s: %v", t.table, err)
}
}
sqlText, args := store.GetSqlScanPage(t.table, cursorHash, cursorName, scanPageSize)
rows, err := tx.QueryContext(ctx, sqlText, args...)
if err != nil {
if t.cachedOnly && isMissingTableError(err) {
tx.ExecContext(ctx, "ROLLBACK TO SAVEPOINT seaweed_scan_bucket")
return nil // a cached-only name whose table was dropped meanwhile
}
return missingTableOr(err, t.table, fmt.Sprintf("scan table %s", t.table))
}
firstPage = false
if err := func() error {
defer rows.Close()
for rows.Next() {
var r row
if err := rows.Scan(&r.dirHash, &r.name, &r.dir, &r.data); err != nil {
return fmt.Errorf("scan table %s: %v", t.table, err)
}
page = append(page, r)
}
return rows.Err()
}(); err != nil {
return err
}
for _, r := range page {
cursorHash, cursorName = r.dirHash, r.name
if r.dirHash != util.HashStringToLong(r.dir) {
continue // a key-value record, see GenDirAndName
}
fullPath := util.NewFullPath(t.base+r.dir, r.name)
if !isStrictlyBelow(fullPath, dirPath) {
continue
}
entry := &filer.Entry{FullPath: fullPath}
if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(r.data)); err != nil {
return fmt.Errorf("scan decode %s: %v", fullPath, err)
}
if err := fn(ctx, entry); err != nil {
return err
}
}
if len(page) < scanPageSize {
return nil
}
}
}
// missingTableOr names the condition when a bucket's table is not there. The
// scan fails closed: from inside the snapshot a table that was never created
// cannot be told from one dropped a moment ago, whose rows the snapshot still
// counts, and treating either as empty would let fsck purge referenced chunks.
func missingTableOr(err error, table, what string) error {
if table != DEFAULT_TABLE && isMissingTableError(err) {
return fmt.Errorf("bucket %s has a directory entry but no table (never created, or dropped during the scan): %v", table, err)
}
return fmt.Errorf("%s: %v", what, err)
}
// isMissingTableError recognizes a missing table by message, as InsertEntry
// recognizes duplicates: PostgreSQL `relation "x" does not exist` (SQLSTATE
// 42P01), MySQL error 1146 (SQLSTATE 42S02), SQLite "no such table".
func isMissingTableError(err error) bool {
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "42p01") ||
(strings.Contains(msg, "relation \"") && strings.Contains(msg, "does not exist")) ||
strings.Contains(msg, "42s02") ||
strings.Contains(msg, "error 1146") ||
strings.Contains(msg, "no such table")
}
// isStrictlyBelow reports whether p is a descendant of dir (not dir itself).
func isStrictlyBelow(p, dir util.FullPath) bool {
if dir == "/" {
return p != "/"
}
return strings.HasPrefix(string(p), string(dir)+"/")
}
@@ -0,0 +1,51 @@
package abstract_sql
import (
"errors"
"testing"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func TestBucketTableFor(t *testing.T) {
cases := map[string]string{
"/": "",
"/buckets": "",
"/buckets/": "",
"/buckets/mybucket": "mybucket",
"/buckets/mybucket/a/b": "mybucket",
"/buckets/.system/owners": "",
"/buckets/Invalid_Bucket/x": "",
"/topics/.system/log": "",
"/bucketsfoo/mybucket": "",
}
for path, want := range cases {
if got := bucketTableFor(util.FullPath(path)); got != want {
t.Errorf("bucketTableFor(%s) = %q, want %q", path, got, want)
}
}
}
// Only a missing table may be skipped; any other schema error must fail the
// scan, or a bucket's live chunks would be reported as orphans.
func TestIsMissingTableError(t *testing.T) {
for _, msg := range []string{
`ERROR: relation "ghost" does not exist (SQLSTATE 42P01)`,
`Error 1146 (42S02): Table 'seaweedfs.ghost' doesn't exist`,
`SQL logic error: no such table: ghost (1)`,
} {
if !isMissingTableError(errors.New(msg)) {
t.Errorf("not recognized as a missing table: %s", msg)
}
}
for _, msg := range []string{
`ERROR: column "directory" does not exist (SQLSTATE 42703)`,
`Error 1054 (42S22): Unknown column 'directory' in 'field list'`,
`SQL logic error: no such column: directory (1)`,
"connection refused",
} {
if isMissingTableError(errors.New(msg)) {
t.Errorf("wrongly taken for a missing table: %s", msg)
}
}
}
+26
View File
@@ -416,6 +416,28 @@ func (f *Filer) RollbackTransaction(ctx context.Context) error {
return f.Store.RollbackTransaction(ctx)
}
type syncWriteKey struct{}
// WithSyncWrite marks the entry writes under ctx durable-on-ack: a store that
// buffers writes by default must flush them before returning.
func WithSyncWrite(ctx context.Context) context.Context {
return context.WithValue(ctx, syncWriteKey{}, true)
}
// SyncWrite reports whether ctx carries a durability request for entry writes.
func SyncWrite(ctx context.Context) bool {
return ctx.Value(syncWriteKey{}) != nil
}
// syncWriteContext flags ctx when the storage rule on p asks for fsync, so the
// store write is held to the same durability as the data path.
func (f *Filer) syncWriteContext(ctx context.Context, p util.FullPath) context.Context {
if f.FilerConf != nil && f.FilerConf.MatchStorageRule(string(p)).Fsync {
return WithSyncWrite(ctx)
}
return ctx
}
// CreateEntry creates or replaces an entry. When existing is non-nil the caller
// has already fetched the current entry at this path under a path lock, and it
// is reused instead of looking the store up again; pass nil to have CreateEntry
@@ -426,6 +448,8 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
return nil
}
ctx = f.syncWriteContext(ctx, entry.FullPath)
if entry.FullPath.IsLongerFileName(maxFilenameLength) {
return filer_pb.ErrEntryNameTooLong
}
@@ -631,6 +655,7 @@ func (f *Filer) DirectoryAttributes(ctx context.Context, dirPath util.FullPath)
// deleted with, so a restore cannot hand back a directory more permissive than the one
// it replaces - including when someone else already put it back with wider ones.
func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath, attrs empty_folder_cleanup.DirectoryAttributes) error {
ctx = f.syncWriteContext(ctx, dirPath)
existing, err := f.FindEntry(ctx, dirPath)
if err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
return err
@@ -680,6 +705,7 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath,
}
func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry, isFromOtherCluster bool) (err error) {
ctx = f.syncWriteContext(ctx, entry.FullPath)
if oldEntry != nil {
entry.Attr.Crtime = oldEntry.Attr.Crtime
if oldEntry.Attr.Inode != 0 {
+2
View File
@@ -122,6 +122,8 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
return nil
}
ctx = f.syncWriteContext(ctx, p)
entry, findErr := f.FindEntry(ctx, p)
if findErr != nil {
return findErr
+16
View File
@@ -44,6 +44,22 @@ type FilerStore interface {
Shutdown()
}
// ErrUnsupportedEntryScan is returned by EntryScanner.ScanEntries, before
// any entry is visited, when the store cannot serve the scan.
var ErrUnsupportedEntryScan = errors.New("entry scan not supported by this filer store")
// EntryScanner is an optional FilerStore capability: visit every entry at and
// below dirPath, in any order, from one consistent snapshot of the store.
//
// A directory walk sees each directory at a different time, so an entry
// renamed from a directory not yet listed into one already listed is never
// seen; volume.fsck then purges its chunks as orphans. A snapshot sees it
// exactly once. The context handed to fn carries the snapshot, so store reads
// made with it (hard-link records) see the same instant.
type EntryScanner interface {
ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *Entry) error) error
}
type BucketAware interface {
OnBucketCreation(bucket string)
OnBucketDeletion(bucket string)
+29
View File
@@ -2,6 +2,7 @@ package filer
import (
"context"
"errors"
"fmt"
"io"
"math"
@@ -19,6 +20,7 @@ import (
var (
_ = VirtualFilerStore(&FilerStoreWrapper{})
_ = Debuggable(&FilerStoreWrapper{})
_ = EntryScanner(&FilerStoreWrapper{})
)
type VirtualFilerStore interface {
@@ -402,6 +404,33 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context,
return lastFileName, err
}
// ScanEntries delegates to the default store. With path-specific stores
// configured, entries below dirPath could live in several stores and one
// snapshot per store is not one snapshot, so the scan is declined. Hard links
// are resolved inside the snapshot; a store error fails the scan (the per-path
// copy can be stale), a missing record falls back to it as listings do.
func (fsw *FilerStoreWrapper) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *Entry) error) error {
if fsw.hasPathSpecificStore {
return ErrUnsupportedEntryScan
}
scanner, ok := fsw.defaultStore.(EntryScanner)
if !ok {
return ErrUnsupportedEntryScan
}
stats.FilerStoreCounter.WithLabelValues(fsw.defaultStore.GetName(), "scan").Inc()
start := time.Now()
defer func() {
stats.FilerStoreHistogram.WithLabelValues(fsw.defaultStore.GetName(), "scan").Observe(time.Since(start).Seconds())
}()
return scanner.ScanEntries(ctx, dirPath, func(scanCtx context.Context, entry *Entry) error {
if err := fsw.maybeReadHardLink(scanCtx, entry); err != nil && !errors.Is(err, ErrKvNotFound) {
return fmt.Errorf("resolve hard link of %s: %w", entry.FullPath, err)
}
filer_pb.AfterEntryDeserialization(entry.GetChunks())
return fn(scanCtx, entry)
})
}
func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
actualStore := fsw.getActualStore(dirPath + "/")
+12 -3
View File
@@ -79,6 +79,15 @@ func (store *LevelDBStore) RollbackTransaction(ctx context.Context) error {
return nil
}
var syncWriteOptions = &opt.WriteOptions{Sync: true}
func writeOptions(ctx context.Context) *opt.WriteOptions {
if filer.SyncWrite(ctx) {
return syncWriteOptions
}
return nil
}
func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
key := genKey(entry.DirAndName())
@@ -91,7 +100,7 @@ func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer.Entry)
value = weed_util.MaybeGzipData(value)
}
err = store.db.Put(key, value, nil)
err = store.db.Put(key, value, writeOptions(ctx))
if err != nil {
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
@@ -168,7 +177,7 @@ func (store *LevelDBStore) FindEntry(ctx context.Context, fullpath weed_util.Ful
func (store *LevelDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
key := genKey(fullpath.DirAndName())
err = store.db.Delete(key, nil)
err = store.db.Delete(key, writeOptions(ctx))
if err != nil {
return fmt.Errorf("delete %s : %v", fullpath, err)
}
@@ -195,7 +204,7 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we
}
iter.Release()
err = store.db.Write(batch, nil)
err = store.db.Write(batch, writeOptions(ctx))
if err != nil {
return fmt.Errorf("delete %s : %v", fullpath, err)
+64 -9
View File
@@ -6,6 +6,7 @@ import (
"crypto/md5"
"fmt"
"io"
"math"
"os"
"github.com/syndtr/goleveldb/leveldb"
@@ -28,6 +29,11 @@ type LevelDB2Store struct {
dbs []*leveldb.DB
dbCount int
ReadOnly bool
blockCacheSizeMB int
writeBufferSizeMB int
blockSizeKB int
compactionTableSizeMB int
}
func (store *LevelDB2Store) GetName() string {
@@ -36,9 +42,54 @@ func (store *LevelDB2Store) GetName() string {
func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
dir := configuration.GetString(prefix + "dir")
store.blockCacheSizeMB = configuration.GetInt(prefix + "block_cache_size_mb")
store.writeBufferSizeMB = configuration.GetInt(prefix + "write_buffer_size_mb")
store.blockSizeKB = configuration.GetInt(prefix + "block_size_kb")
store.compactionTableSizeMB = configuration.GetInt(prefix + "compaction_table_size_mb")
if err := store.validateSizeOptions(); err != nil {
return err
}
return store.initialize(dir, 8)
}
// validateSizeOptions rejects sizes too large to convert to bytes without
// overflowing int on this platform.
func (store *LevelDB2Store) validateSizeOptions() error {
for name, kv := range map[string][2]int{
"block_cache_size_mb": {store.blockCacheSizeMB, 1 << 20},
"write_buffer_size_mb": {store.writeBufferSizeMB, 1 << 20},
"block_size_kb": {store.blockSizeKB, 1 << 10},
"compaction_table_size_mb": {store.compactionTableSizeMB, 1 << 20},
} {
if kv[0] > math.MaxInt/kv[1] {
return fmt.Errorf("leveldb2 option %s=%d overflows int", name, kv[0])
}
}
return nil
}
func (store *LevelDB2Store) buildOptions() *opt.Options {
opts := &opt.Options{
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
Filter: filter.NewBloomFilter(8), // false positive rate 0.02
ReadOnly: store.ReadOnly,
}
if store.blockCacheSizeMB > 0 {
opts.BlockCacheCapacity = store.blockCacheSizeMB * 1024 * 1024
}
if store.writeBufferSizeMB > 0 {
opts.WriteBuffer = store.writeBufferSizeMB * 1024 * 1024
}
if store.blockSizeKB > 0 {
opts.BlockSize = store.blockSizeKB * 1024
}
if store.compactionTableSizeMB > 0 {
opts.CompactionTableSize = store.compactionTableSizeMB * 1024 * 1024
}
return opts
}
func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
glog.Infof("filer store leveldb2 dir: %s", dir)
os.MkdirAll(dir, 0755)
@@ -46,12 +97,7 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
}
opts := &opt.Options{
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
Filter: filter.NewBloomFilter(8), // false positive rate 0.02
ReadOnly: store.ReadOnly,
}
opts := store.buildOptions()
for d := 0; d < dbCount; d++ {
dbFolder := fmt.Sprintf("%s/%02d", dir, d)
@@ -81,6 +127,15 @@ func (store *LevelDB2Store) RollbackTransaction(ctx context.Context) error {
return nil
}
var syncWriteOptions = &opt.WriteOptions{Sync: true}
func writeOptions(ctx context.Context) *opt.WriteOptions {
if filer.SyncWrite(ctx) {
return syncWriteOptions
}
return nil
}
func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
dir, name := entry.DirAndName()
key, partitionId := genKey(dir, name, store.dbCount)
@@ -94,7 +149,7 @@ func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer.Entry)
value = weed_util.MaybeGzipData(value)
}
err = store.dbs[partitionId].Put(key, value, nil)
err = store.dbs[partitionId].Put(key, value, writeOptions(ctx))
if err != nil {
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
@@ -140,7 +195,7 @@ func (store *LevelDB2Store) DeleteEntry(ctx context.Context, fullpath weed_util.
dir, name := fullpath.DirAndName()
key, partitionId := genKey(dir, name, store.dbCount)
err = store.dbs[partitionId].Delete(key, nil)
err = store.dbs[partitionId].Delete(key, writeOptions(ctx))
if err != nil {
return fmt.Errorf("delete %s : %v", fullpath, err)
}
@@ -167,7 +222,7 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w
}
iter.Release()
err = store.dbs[partitionId].Write(batch, nil)
err = store.dbs[partitionId].Write(batch, writeOptions(ctx))
if err != nil {
return fmt.Errorf("delete %s : %v", fullpath, err)
@@ -2,6 +2,8 @@ package leveldb
import (
"context"
"math"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/filer"
@@ -62,6 +64,93 @@ func TestCreateAndFind(t *testing.T) {
}
type fakeConfiguration map[string]interface{}
func (c fakeConfiguration) GetString(key string) string {
value, _ := c[key].(string)
return value
}
func (c fakeConfiguration) GetBool(key string) bool {
value, _ := c[key].(bool)
return value
}
func (c fakeConfiguration) GetInt(key string) int {
value, _ := c[key].(int)
return value
}
func (c fakeConfiguration) GetStringSlice(key string) []string {
value, _ := c[key].([]string)
return value
}
func (c fakeConfiguration) SetDefault(key string, value interface{}) {
if _, found := c[key]; !found {
c[key] = value
}
}
func TestInitializeOptions(t *testing.T) {
store := &LevelDB2Store{}
if err := store.Initialize(fakeConfiguration{
"leveldb2.dir": t.TempDir(),
"leveldb2.block_cache_size_mb": 64,
"leveldb2.write_buffer_size_mb": 32,
"leveldb2.block_size_kb": 16,
"leveldb2.compaction_table_size_mb": 8,
}, "leveldb2."); err != nil {
t.Fatalf("initialize: %v", err)
}
defer store.Shutdown()
opts := store.buildOptions()
if opts.BlockCacheCapacity != 64*1024*1024 {
t.Errorf("block cache %d, want 64MiB", opts.BlockCacheCapacity)
}
if opts.WriteBuffer != 32*1024*1024 {
t.Errorf("write buffer %d, want 32MiB", opts.WriteBuffer)
}
if opts.BlockSize != 16*1024 {
t.Errorf("block size %d, want 16KiB", opts.BlockSize)
}
if opts.CompactionTableSize != 8*1024*1024 {
t.Errorf("compaction table size %d, want 8MiB", opts.CompactionTableSize)
}
}
func TestInitializeDefaultOptions(t *testing.T) {
store := &LevelDB2Store{}
if err := store.Initialize(fakeConfiguration{
"leveldb2.dir": t.TempDir(),
"leveldb2.block_cache_size_mb": -1,
"leveldb2.write_buffer_size_mb": 0,
}, "leveldb2."); err != nil {
t.Fatalf("initialize: %v", err)
}
defer store.Shutdown()
opts := store.buildOptions()
if opts.BlockCacheCapacity != 32*1024*1024 {
t.Errorf("block cache %d, want the 32MiB default", opts.BlockCacheCapacity)
}
if opts.WriteBuffer != 16*1024*1024 {
t.Errorf("write buffer %d, want the 16MiB default", opts.WriteBuffer)
}
}
func TestInitializeRejectsOversizedOption(t *testing.T) {
store := &LevelDB2Store{}
err := store.Initialize(fakeConfiguration{
"leveldb2.dir": t.TempDir(),
"leveldb2.write_buffer_size_mb": math.MaxInt/(1<<20) + 1,
}, "leveldb2.")
if err == nil || !strings.Contains(err.Error(), "write_buffer_size_mb") {
t.Fatalf("expected overflow rejection, got %v", err)
}
}
func TestEmptyRoot(t *testing.T) {
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
dir := t.TempDir()
+68 -18
View File
@@ -6,6 +6,7 @@ import (
"crypto/md5"
"fmt"
"io"
"math"
"os"
"strings"
"sync"
@@ -35,6 +36,11 @@ type LevelDB3Store struct {
dbs map[string]*leveldb.DB
dbsLock sync.RWMutex
ReadOnly bool
blockCacheSizeMB int
writeBufferSizeMB int
blockSizeKB int
compactionTableSizeMB int
}
func (store *LevelDB3Store) GetName() string {
@@ -43,9 +49,58 @@ func (store *LevelDB3Store) GetName() string {
func (store *LevelDB3Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
dir := configuration.GetString(prefix + "dir")
store.blockCacheSizeMB = configuration.GetInt(prefix + "block_cache_size_mb")
store.writeBufferSizeMB = configuration.GetInt(prefix + "write_buffer_size_mb")
store.blockSizeKB = configuration.GetInt(prefix + "block_size_kb")
store.compactionTableSizeMB = configuration.GetInt(prefix + "compaction_table_size_mb")
if err := store.validateSizeOptions(); err != nil {
return err
}
return store.initialize(dir)
}
// validateSizeOptions rejects sizes too large to convert to bytes without
// overflowing int on this platform.
func (store *LevelDB3Store) validateSizeOptions() error {
for name, kv := range map[string][2]int{
"block_cache_size_mb": {store.blockCacheSizeMB, 1 << 20},
"write_buffer_size_mb": {store.writeBufferSizeMB, 1 << 20},
"block_size_kb": {store.blockSizeKB, 1 << 10},
"compaction_table_size_mb": {store.compactionTableSizeMB, 1 << 20},
} {
if kv[0] > math.MaxInt/kv[1] {
return fmt.Errorf("leveldb3 option %s=%d overflows int", name, kv[0])
}
}
return nil
}
func (store *LevelDB3Store) buildOptions(name string) *opt.Options {
opts := &opt.Options{
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
Filter: filter.NewBloomFilter(8),
ReadOnly: store.ReadOnly,
}
if name != DEFAULT {
opts.BlockCacheCapacity = 16 * 1024 * 1024 // default value is 8MiB
opts.WriteBuffer = 8 * 1024 * 1024 // default value is 4MiB
}
if store.blockCacheSizeMB > 0 {
opts.BlockCacheCapacity = store.blockCacheSizeMB * 1024 * 1024
}
if store.writeBufferSizeMB > 0 {
opts.WriteBuffer = store.writeBufferSizeMB * 1024 * 1024
}
if store.blockSizeKB > 0 {
opts.BlockSize = store.blockSizeKB * 1024
}
if store.compactionTableSizeMB > 0 {
opts.CompactionTableSize = store.compactionTableSizeMB * 1024 * 1024
}
return opts
}
func (store *LevelDB3Store) initialize(dir string) (err error) {
glog.Infof("filer store leveldb3 dir: %s", dir)
os.MkdirAll(dir, 0755)
@@ -65,21 +120,7 @@ func (store *LevelDB3Store) initialize(dir string) (err error) {
}
func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) {
bloom := filter.NewBloomFilter(8) // false positive rate 0.02
opts := &opt.Options{
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
Filter: bloom,
ReadOnly: store.ReadOnly,
}
if name != DEFAULT {
opts = &opt.Options{
BlockCacheCapacity: 16 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 8 * 1024 * 1024, // default value is 4MiB
Filter: bloom,
ReadOnly: store.ReadOnly,
}
}
opts := store.buildOptions(name)
dbFolder := fmt.Sprintf("%s/%s", store.dir, name)
os.MkdirAll(dbFolder, 0755)
@@ -180,6 +221,15 @@ func (store *LevelDB3Store) RollbackTransaction(ctx context.Context) error {
return nil
}
var syncWriteOptions = &opt.WriteOptions{Sync: true}
func writeOptions(ctx context.Context) *opt.WriteOptions {
if filer.SyncWrite(ctx) {
return syncWriteOptions
}
return nil
}
func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
db, _, shortPath, err := store.findDB(entry.FullPath, false)
@@ -199,7 +249,7 @@ func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry)
value = weed_util.MaybeGzipData(value)
}
err = db.Put(key, value, nil)
err = db.Put(key, value, writeOptions(ctx))
if err != nil {
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
@@ -257,7 +307,7 @@ func (store *LevelDB3Store) DeleteEntry(ctx context.Context, fullpath weed_util.
dir, name := shortPath.DirAndName()
key := genKey(dir, name)
err = db.Delete(key, nil)
err = db.Delete(key, writeOptions(ctx))
if err != nil {
return fmt.Errorf("delete %s : %v", fullpath, err)
}
@@ -298,7 +348,7 @@ func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath w
}
iter.Release()
err = db.Write(batch, nil)
err = db.Write(batch, writeOptions(ctx))
if err != nil {
return fmt.Errorf("delete %s : %v", fullpath, err)
@@ -2,6 +2,8 @@ package leveldb
import (
"context"
"math"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/filer"
@@ -62,6 +64,103 @@ func TestCreateAndFind(t *testing.T) {
}
type fakeConfiguration map[string]interface{}
func (c fakeConfiguration) GetString(key string) string {
value, _ := c[key].(string)
return value
}
func (c fakeConfiguration) GetBool(key string) bool {
value, _ := c[key].(bool)
return value
}
func (c fakeConfiguration) GetInt(key string) int {
value, _ := c[key].(int)
return value
}
func (c fakeConfiguration) GetStringSlice(key string) []string {
value, _ := c[key].([]string)
return value
}
func (c fakeConfiguration) SetDefault(key string, value interface{}) {
if _, found := c[key]; !found {
c[key] = value
}
}
func TestInitializeOptions(t *testing.T) {
store := &LevelDB3Store{}
if err := store.Initialize(fakeConfiguration{
"leveldb3.dir": t.TempDir(),
"leveldb3.block_cache_size_mb": 64,
"leveldb3.write_buffer_size_mb": 32,
"leveldb3.block_size_kb": 16,
"leveldb3.compaction_table_size_mb": 8,
}, "leveldb3."); err != nil {
t.Fatalf("initialize: %v", err)
}
defer store.Shutdown()
for _, name := range []string{DEFAULT, "bucket-a"} {
opts := store.buildOptions(name)
if opts.BlockCacheCapacity != 64*1024*1024 {
t.Errorf("%s block cache %d, want 64MiB", name, opts.BlockCacheCapacity)
}
if opts.WriteBuffer != 32*1024*1024 {
t.Errorf("%s write buffer %d, want 32MiB", name, opts.WriteBuffer)
}
if opts.BlockSize != 16*1024 {
t.Errorf("%s block size %d, want 16KiB", name, opts.BlockSize)
}
if opts.CompactionTableSize != 8*1024*1024 {
t.Errorf("%s compaction table size %d, want 8MiB", name, opts.CompactionTableSize)
}
}
}
func TestInitializeDefaultOptions(t *testing.T) {
store := &LevelDB3Store{}
if err := store.Initialize(fakeConfiguration{
"leveldb3.dir": t.TempDir(),
"leveldb3.block_cache_size_mb": -1,
"leveldb3.write_buffer_size_mb": 0,
}, "leveldb3."); err != nil {
t.Fatalf("initialize: %v", err)
}
defer store.Shutdown()
opts := store.buildOptions(DEFAULT)
if opts.BlockCacheCapacity != 32*1024*1024 {
t.Errorf("block cache %d, want the 32MiB default", opts.BlockCacheCapacity)
}
if opts.WriteBuffer != 16*1024*1024 {
t.Errorf("write buffer %d, want the 16MiB default", opts.WriteBuffer)
}
bucketOpts := store.buildOptions("bucket-a")
if bucketOpts.BlockCacheCapacity != 16*1024*1024 {
t.Errorf("bucket block cache %d, want the 16MiB default", bucketOpts.BlockCacheCapacity)
}
if bucketOpts.WriteBuffer != 8*1024*1024 {
t.Errorf("bucket write buffer %d, want the 8MiB default", bucketOpts.WriteBuffer)
}
}
func TestInitializeRejectsOversizedOption(t *testing.T) {
store := &LevelDB3Store{}
err := store.Initialize(fakeConfiguration{
"leveldb3.dir": t.TempDir(),
"leveldb3.write_buffer_size_mb": math.MaxInt/(1<<20) + 1,
}, "leveldb3.")
if err == nil || !strings.Contains(err.Error(), "write_buffer_size_mb") {
t.Fatalf("expected overflow rejection, got %v", err)
}
}
func TestEmptyRoot(t *testing.T) {
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
dir := t.TempDir()
+9
View File
@@ -62,6 +62,15 @@ func (gen *SqlGenMysql) nameExpr() string {
return "`name`"
}
// GetSqlScanPage spells the cursor as two primary-key ranges: MySQL 5.7 does
// not turn a row-value comparison into a range seek. The name keeps the
// column's own collation (BINARY would filesort every page); the scan promises
// no lexical order. SQLite overrides this method.
func (gen *SqlGenMysql) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) {
return fmt.Sprintf("SELECT `dirhash`, `name`, `directory`, `meta` FROM `%s` WHERE `dirhash` > ? OR (`dirhash` = ? AND `name` > ?) ORDER BY `dirhash` ASC, `name` ASC LIMIT ?", tableName),
[]interface{}{dirhash, dirhash, name, limit}
}
func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string {
name := gen.nameExpr()
return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s > ? AND `directory` = ? AND %s LIKE ? ESCAPE '!' ORDER BY %s ASC LIMIT ?", tableName, name, name, name)
+8
View File
@@ -75,6 +75,14 @@ func (gen *SqlGenPostgres) nameExpr() string {
return "name"
}
// GetSqlScanPage uses a row-value comparison, one primary-key range seek on
// PostgreSQL. The name keeps the column's own collation (COLLATE "C" would
// sort every page); the scan promises no lexical order.
func (gen *SqlGenPostgres) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) {
return fmt.Sprintf(`SELECT dirhash, name, directory, meta FROM "%s" WHERE (dirhash, name) > ($1, $2) ORDER BY dirhash ASC, name ASC LIMIT $3`, tableName),
[]interface{}{dirhash, name, limit}
}
func (gen *SqlGenPostgres) GetSqlListExclusive(tableName string) string {
name := gen.nameExpr()
return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>$2 AND directory=$3 AND %s like $4 ESCAPE '!' ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)
+32 -2
View File
@@ -30,6 +30,31 @@ func (store *SqliteStore) GetName() string {
return "sqlite"
}
// sqliteSqlGen is the MySQL generator with SQLite's own page cursor: SQLite
// seeks a row-value comparison on the primary key but scans the OR form.
type sqliteSqlGen struct {
*mysql.SqlGenMysql
}
func (gen sqliteSqlGen) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) {
return fmt.Sprintf("SELECT `dirhash`, `name`, `directory`, `meta` FROM `%s` WHERE (`dirhash`, `name`) > (?, ?) ORDER BY `dirhash` ASC, `name` ASC LIMIT ?", tableName),
[]interface{}{dirhash, name, limit}
}
// ScanEntries offers the snapshot scan only in WAL mode: with a rollback
// journal a long read transaction holds a shared lock and writers fail after
// the busy timeout.
func (store *SqliteStore) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error {
var mode string
if err := store.DB.QueryRowContext(ctx, "PRAGMA journal_mode").Scan(&mode); err != nil {
return fmt.Errorf("%w: journal_mode: %v", filer.ErrUnsupportedEntryScan, err)
}
if !strings.EqualFold(mode, "wal") {
return fmt.Errorf("%w: sqlite journal_mode is %q, the scan needs wal", filer.ErrUnsupportedEntryScan, mode)
}
return store.AbstractSqlStore.ScanEntries(ctx, dirPath, fn)
}
func (store *SqliteStore) Initialize(configuration util.Configuration, prefix string) (err error) {
dbFile := configuration.GetString(prefix + "dbFile")
createTable := `CREATE TABLE IF NOT EXISTS "%s" (
@@ -69,11 +94,11 @@ func sqliteDSN(dbFile string) string {
func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (err error) {
store.SupportBucketTable = true
store.SqlGenerator = &mysql.SqlGenMysql{
store.SqlGenerator = sqliteSqlGen{&mysql.SqlGenMysql{
CreateTableSqlTemplate: createTable,
DropTableSqlTemplate: "drop table if exists `%s`",
UpsertQueryTemplate: upsertQuery,
}
}}
dsn := sqliteDSN(dbFile)
@@ -94,6 +119,11 @@ func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (e
}, 1, 1, 0); err != nil {
return err
}
// The main pool is one connection, which a long scan must not hold.
if store.ScanDB, err = sql.Open("sqlite", dsn); err != nil {
return fmt.Errorf("open scan connection to %s: %v", dbFile, err)
}
store.ScanDB.SetMaxOpenConns(1)
if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {
return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)
+288
View File
@@ -0,0 +1,288 @@
//go:build (linux || darwin || windows) && sqlite
package sqlite
import (
"context"
"errors"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// A rename committed while a scan is running is seen under exactly one of its
// names. The scan reads from one snapshot, so unlike a directory walk it can
// neither miss the entry when it moves into a directory already listed, nor
// count it twice when it moves into one not listed yet.
func TestScanEntriesSeesConcurrentRenameExactlyOnce(t *testing.T) {
dbFile := filepath.Join(t.TempDir(), "filer.db")
reader := &SqliteStore{}
if err := reader.initialize(dbFile, createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer reader.Shutdown()
// WAL lets the writer commit while the scan's read transaction is open.
if _, err := reader.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
t.Fatal(err)
}
writer := &SqliteStore{}
if err := writer.initialize(dbFile, createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer writer.Shutdown()
ctx := context.Background()
mk := func(p string) *filer.Entry {
return &filer.Entry{FullPath: util.FullPath(p), Attr: filer.Attr{Mode: 0644, Mtime: time.Now(), Crtime: time.Now()}}
}
for _, p := range []string{"/ckpt/a", "/ckpt/b", "/ckpt/tmp/x"} {
if err := reader.InsertEntry(ctx, mk(p)); err != nil {
t.Fatal(err)
}
}
defer func(n int) { abstract_sql.SetScanPageSizeForTest(n) }(abstract_sql.SetScanPageSizeForTest(2))
seen := map[string]int{}
renamed := false
err := reader.ScanEntries(ctx, "/", func(_ context.Context, e *filer.Entry) error {
seen[string(e.FullPath)]++
if renamed {
return nil
}
renamed = true
// /ckpt/tmp/x -> /ckpt/x, committed by another connection mid-scan.
if err := writer.InsertEntry(ctx, mk("/ckpt/x")); err != nil {
return err
}
return writer.DeleteEntry(ctx, util.FullPath("/ckpt/tmp/x"))
})
if err != nil {
t.Fatal(err)
}
if !renamed {
t.Fatal("scan yielded nothing")
}
if seen["/ckpt/a"] != 1 || seen["/ckpt/b"] != 1 {
t.Errorf("untouched entries seen %d and %d times, want once each", seen["/ckpt/a"], seen["/ckpt/b"])
}
if old, new := seen["/ckpt/tmp/x"], seen["/ckpt/x"]; old+new != 1 {
t.Errorf("renamed entry seen %d time(s) under the old name and %d under the new, want exactly one in total", old, new)
} else if old != 1 {
t.Errorf("the rename committed after the snapshot was taken, so the scan must show the old name")
}
if _, err := reader.FindEntry(ctx, util.FullPath("/ckpt/x")); err != nil {
t.Errorf("once the scan is over the new name must be visible: %v", err)
}
}
// Rows that are not entries, key-value records and a regular file under
// /buckets with a bucket-like name, are stepped over; a scan rooted at a bucket
// yields the bucket's own row (default table) and its contents (bucket table).
func TestScanEntriesRowsAndBucketTables(t *testing.T) {
store := &SqliteStore{}
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer store.Shutdown()
if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
t.Fatal(err)
}
ctx := context.Background()
now := time.Now()
// The filer creates bucket tables on bucket creation; the store layer alone does not.
if err := store.CreateTable(ctx, "realbucket"); err != nil {
t.Fatal(err)
}
for _, e := range []*filer.Entry{
{FullPath: "/buckets", Attr: filer.Attr{Mode: 0755 | os.ModeDir, Mtime: now, Crtime: now}},
{FullPath: "/buckets/realbucket", Attr: filer.Attr{Mode: 0755 | os.ModeDir, Mtime: now, Crtime: now}},
{FullPath: "/buckets/realbucket/f1", Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}},
{FullPath: "/buckets/readme", Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}},
{FullPath: "/topics/t1", Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}},
} {
if err := store.InsertEntry(ctx, e); err != nil {
t.Fatal(err)
}
}
for _, key := range []string{"filer.store.id", "/looks/like/a/path", "x"} {
if err := store.KvPut(ctx, []byte(key), []byte("opaque value, not an Entry")); err != nil {
t.Fatal(err)
}
}
collect := func(root string) []string {
var got []string
if err := store.ScanEntries(ctx, util.FullPath(root), func(_ context.Context, e *filer.Entry) error {
got = append(got, string(e.FullPath))
return nil
}); err != nil {
t.Fatalf("scan %s: %v", root, err)
}
sort.Strings(got)
return got
}
if got, want := collect("/"), []string{"/buckets", "/buckets/readme", "/buckets/realbucket", "/buckets/realbucket/f1", "/topics/t1"}; !reflect.DeepEqual(got, want) {
t.Errorf("scan /: got %v, want %v", got, want)
}
if got, want := collect("/buckets/realbucket"), []string{"/buckets/realbucket", "/buckets/realbucket/f1"}; !reflect.DeepEqual(got, want) {
t.Errorf("scan /buckets/realbucket: got %v, want %v", got, want)
}
// A bucket directory whose table does not exist fails the scan: from the
// snapshot it cannot be told from a table dropped a moment ago.
if err := store.InsertEntry(ctx, &filer.Entry{FullPath: "/buckets/ghostbucket", Attr: filer.Attr{Mode: 0755 | os.ModeDir, Mtime: now, Crtime: now}}); err != nil {
t.Fatal(err)
}
err := store.ScanEntries(ctx, "/", func(context.Context, *filer.Entry) error { return nil })
if err == nil || !strings.Contains(err.Error(), "ghostbucket") {
t.Fatalf("scan over a bucket without a table must fail naming it, got %v", err)
}
}
// Without WAL the store declines the scan rather than block writers for its
// whole duration.
func TestScanEntriesDeclinedWithoutWAL(t *testing.T) {
store := &SqliteStore{}
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer store.Shutdown()
err := store.ScanEntries(context.Background(), "/", func(context.Context, *filer.Entry) error { return nil })
if !errors.Is(err, filer.ErrUnsupportedEntryScan) {
t.Fatalf("err = %v, want ErrUnsupportedEntryScan", err)
}
}
// A store whose pool is one connection, and no scan connection of its own,
// declines rather than hold that connection for the whole scan.
func TestScanEntriesDeclinedOnSingleConnectionPool(t *testing.T) {
store := &SqliteStore{}
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer store.Shutdown()
if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
t.Fatal(err)
}
store.ScanDB.Close()
store.ScanDB = nil
err := store.ScanEntries(context.Background(), "/", func(context.Context, *filer.Entry) error { return nil })
if !errors.Is(err, filer.ErrUnsupportedEntryScan) {
t.Fatalf("err = %v, want ErrUnsupportedEntryScan", err)
}
}
// A hard-linked entry's chunks live in the shared key-value record, not in
// its path row; the wrapper resolves them from inside the snapshot, on the
// single connection the scan holds.
func TestScanEntriesResolvesHardLinks(t *testing.T) {
store := &SqliteStore{}
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer store.Shutdown()
if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
t.Fatal(err)
}
wrapper := filer.NewFilerStoreWrapper(store)
ctx := context.Background()
now := time.Now()
linked := &filer.Entry{
FullPath: "/buckets/hlbucket/f0001",
Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now},
HardLinkId: filer.HardLinkId("hardlink-00000001"),
HardLinkCounter: 2,
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 4096}},
}
if err := wrapper.InsertEntry(ctx, linked); err != nil {
t.Fatal(err)
}
var got *filer.Entry
if err := wrapper.ScanEntries(ctx, "/", func(_ context.Context, e *filer.Entry) error {
if e.FullPath == linked.FullPath {
got = e
}
return nil
}); err != nil {
t.Fatal(err)
}
if got == nil || len(got.GetChunks()) != 1 || got.GetChunks()[0].FileId != "3,01637037d6" || got.HardLinkCounter != 2 {
t.Fatalf("hard-linked entry not resolved from its key-value record: %+v", got)
}
// Once the record is gone the path row is all there is; the scan hands it
// over instead of failing on that entry every time it runs.
if err := store.KvDelete(ctx, []byte(linked.HardLinkId)); err != nil {
t.Fatal(err)
}
got = nil
if err := wrapper.ScanEntries(ctx, "/", func(_ context.Context, e *filer.Entry) error {
if e.FullPath == linked.FullPath {
got = e
}
return nil
}); err != nil {
t.Fatalf("a missing hard-link record must not fail the scan: %v", err)
}
if got == nil {
t.Fatal("entry with a missing hard-link record not handed over")
}
}
// The keyset page must be a primary-key seek; an OR-shaped cursor predicate
// makes SQLite scan the table from the start for every page.
func TestScanPageQuerySeeksPrimaryKey(t *testing.T) {
store := &SqliteStore{}
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer store.Shutdown()
sqlText, args := store.GetSqlScanPage("filemeta", int64(-1), "", 10)
rows, err := store.DB.Query("EXPLAIN QUERY PLAN "+sqlText, args...)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
var plan []string
for rows.Next() {
var id, parent, notused int
var detail string
if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
t.Fatal(err)
}
plan = append(plan, detail)
}
joined := strings.Join(plan, " | ")
if !strings.Contains(joined, "SEARCH") || strings.Contains(joined, "SCAN") {
t.Fatalf("page query does not seek the primary key: %s", joined)
}
}
// A bucket name only the process cache remembers may have its table dropped
// meanwhile; the scan skips it instead of failing.
func TestScanEntriesSkipsCachedOnlyBucketTable(t *testing.T) {
store := &SqliteStore{}
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
t.Fatal(err)
}
defer store.Shutdown()
if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
t.Fatal(err)
}
store.OnBucketCreation("gonebucket") // the table is never created
if err := store.ScanEntries(context.Background(), "/", func(context.Context, *filer.Entry) error { return nil }); err != nil {
t.Fatalf("cached-only bucket with a dropped table must not fail the scan: %v", err)
}
}