mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-17 09:17:39 +01:00
Merge branch 'master' of https://github.com/seaweedfs/seaweedfs into nfs-enable-inode-index
This commit is contained in:
commit
347d02cf9e
213 files changed
+16701
-1571
No files matched your search
@@ -23,14 +23,20 @@ type SqlGenerator interface {
|
||||
GetSqlDeleteFolderChildren(tableName string) string
|
||||
GetSqlListExclusive(tableName string) string
|
||||
GetSqlListInclusive(tableName string) string
|
||||
// GetSqlScanPage returns the query and arguments selecting dirhash, name,
|
||||
// directory and meta of up to limit rows after the (dirhash, name) cursor.
|
||||
GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{})
|
||||
GetSqlCreateTable(tableName string) string
|
||||
GetSqlDropTable(tableName string) string
|
||||
}
|
||||
|
||||
type AbstractSqlStore struct {
|
||||
SqlGenerator
|
||||
DB *sql.DB
|
||||
KvDB *sql.DB
|
||||
DB *sql.DB
|
||||
KvDB *sql.DB
|
||||
// ScanDB, when set, is the pool snapshot scans run on (SQLite's main pool
|
||||
// is a single connection that a long scan must not hold).
|
||||
ScanDB *sql.DB
|
||||
SupportBucketTable bool
|
||||
SkipDDL bool
|
||||
dbs map[string]bool
|
||||
@@ -457,6 +463,9 @@ func (store *AbstractSqlStore) Shutdown() {
|
||||
if store.KvDB != nil {
|
||||
store.KvDB.Close()
|
||||
}
|
||||
if store.ScanDB != nil {
|
||||
store.ScanDB.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func isValidBucket(bucket string) bool {
|
||||
|
||||
@@ -0,0 +1,368 @@
|
||||
package abstract_sql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var _ filer.EntryScanner = (*AbstractSqlStore)(nil)
|
||||
|
||||
// scanTable is one table to read during a scan together with the path its
|
||||
// directory column is relative to: "" for the default table, whose rows hold
|
||||
// full directories, and "/buckets/<bucket>" for a per-bucket table.
|
||||
type scanTable struct {
|
||||
table string
|
||||
base string
|
||||
// cachedOnly marks a table that only a stale bucket cache remembers: its
|
||||
// bucket directory was already gone from the snapshot, so a missing table
|
||||
// means it was dropped meanwhile, not that rows are absent.
|
||||
cachedOnly bool
|
||||
}
|
||||
|
||||
// scanPageSize is how many rows one keyset page of a scan fetches.
|
||||
var scanPageSize = 1000
|
||||
|
||||
// SetScanPageSizeForTest lets tests drive the keyset cursor across pages. It
|
||||
// returns the previous size.
|
||||
func SetScanPageSizeForTest(n int) int {
|
||||
prev := scanPageSize
|
||||
scanPageSize = n
|
||||
return prev
|
||||
}
|
||||
|
||||
// ScanEntries implements filer.EntryScanner: every table that can hold an
|
||||
// entry below dirPath is read inside one REPEATABLE READ, read-only
|
||||
// transaction, in primary-key pages. All rows are seen as of one instant; a
|
||||
// rename committed meanwhile shows up under exactly one of its names. The
|
||||
// transaction takes no row locks and does not block writers; it does hold the
|
||||
// tables it has read against DDL, so a bucket deleted while the scan runs
|
||||
// drops its table once the scan ends. A page's cursor is closed before its
|
||||
// entries go to fn, so fn may read the store through the context it receives
|
||||
// (which carries the transaction) on the same connection.
|
||||
func (store *AbstractSqlStore) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error {
|
||||
// Read before the transaction holds a connection: OnBucketCreation keeps
|
||||
// dbsLock while it waits for a connection of its own, so taking the lock
|
||||
// while holding a connection could deadlock on an exhausted pool.
|
||||
cachedBuckets := store.cachedBucketNames()
|
||||
|
||||
tx, err := store.beginSnapshotTx(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Read-only: nothing to commit, and a rollback releases the snapshot.
|
||||
defer tx.Rollback()
|
||||
|
||||
tables, err := store.scanTables(ctx, tx, dirPath, cachedBuckets)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Reads made through this context (getTxOrDB, kvDB) use the transaction
|
||||
// and therefore the snapshot.
|
||||
txCtx := context.WithValue(ctx, "tx", tx)
|
||||
// The root's own row may live in a table the scan does not read (a
|
||||
// bucket directory is in the default table, its contents in the bucket's).
|
||||
if root, err := store.readRowInSnapshot(txCtx, tx, dirPath); err != nil {
|
||||
return err
|
||||
} else if root != nil {
|
||||
if err := fn(txCtx, root); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, t := range tables {
|
||||
if err := store.scanOneTable(txCtx, tx, t, dirPath, fn); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readRowInSnapshot reads one entry's row inside the transaction, routing to
|
||||
// the table getTxOrDB would use but without its bucket-table creation. A
|
||||
// missing row is nil.
|
||||
func (store *AbstractSqlStore) readRowInSnapshot(ctx context.Context, tx *sql.Tx, p util.FullPath) (*filer.Entry, error) {
|
||||
if p == "/" {
|
||||
return nil, nil
|
||||
}
|
||||
parent, name := p.DirAndName()
|
||||
table, shortDir := DEFAULT_TABLE, parent
|
||||
if store.SupportBucketTable {
|
||||
if bucket := bucketTableFor(util.FullPath(parent)); bucket != "" {
|
||||
table = bucket
|
||||
if shortDir = strings.TrimPrefix(parent, bucketBase(bucket)); shortDir == "" {
|
||||
shortDir = "/"
|
||||
}
|
||||
}
|
||||
}
|
||||
var data []byte
|
||||
err := tx.QueryRowContext(ctx, store.GetSqlFind(table), util.HashStringToLong(shortDir), name, shortDir).Scan(&data)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, missingTableOr(err, table, fmt.Sprintf("read %s", p))
|
||||
}
|
||||
entry := &filer.Entry{FullPath: p}
|
||||
if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
|
||||
return nil, fmt.Errorf("decode %s: %v", p, err)
|
||||
}
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
// beginSnapshotTx starts the read-only REPEATABLE READ transaction that gives
|
||||
// one snapshot across statements. Nothing weaker is tried: under READ
|
||||
// COMMITTED each page would see a different snapshot, and the stream would
|
||||
// still be advertised as one.
|
||||
func (store *AbstractSqlStore) beginSnapshotTx(ctx context.Context) (*sql.Tx, error) {
|
||||
db := store.DB
|
||||
if store.ScanDB != nil {
|
||||
db = store.ScanDB
|
||||
} else if db.Stats().MaxOpenConnections == 1 {
|
||||
// The scan would hold the filer's only connection for its whole run.
|
||||
return nil, fmt.Errorf("%w: the store's single connection cannot be held for a scan", filer.ErrUnsupportedEntryScan)
|
||||
}
|
||||
// A BeginTx failure (a down database) must fail the traversal, not route it
|
||||
// to the walk: a recovered database lets the walk read renamed-away entries
|
||||
// as absent, the very loss this scan exists to prevent.
|
||||
tx, err := db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("begin snapshot: %v", err)
|
||||
}
|
||||
return tx, nil
|
||||
}
|
||||
|
||||
func (store *AbstractSqlStore) cachedBucketNames() []string {
|
||||
store.dbsLock.Lock()
|
||||
defer store.dbsLock.Unlock()
|
||||
names := make([]string, 0, len(store.dbs))
|
||||
for name := range store.dbs {
|
||||
names = append(names, name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// scanTables picks the tables to read for dirPath: the default table alone,
|
||||
// one bucket's table for a path inside that bucket, or the default table plus
|
||||
// every bucket table (enumerated from /buckets inside the snapshot).
|
||||
func (store *AbstractSqlStore) scanTables(ctx context.Context, tx *sql.Tx, dirPath util.FullPath, cachedBuckets []string) ([]scanTable, error) {
|
||||
if !store.SupportBucketTable {
|
||||
return []scanTable{{table: DEFAULT_TABLE}}, nil
|
||||
}
|
||||
if bucket := bucketTableFor(dirPath); bucket != "" {
|
||||
return []scanTable{{table: bucket, base: bucketBase(bucket)}}, nil
|
||||
}
|
||||
tables := []scanTable{{table: DEFAULT_TABLE}}
|
||||
if dirPath != "/" && string(dirPath) != bucketsRoot {
|
||||
// Outside every bucket (e.g. /topics, or an internal /buckets/.system
|
||||
// folder): everything below dirPath is in the default table.
|
||||
return tables, nil
|
||||
}
|
||||
buckets, err := store.listBucketNames(ctx, tx, cachedBuckets)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, bucket := range buckets.names {
|
||||
tables = append(tables, scanTable{table: bucket, base: bucketBase(bucket), cachedOnly: !buckets.seen[bucket]})
|
||||
}
|
||||
return tables, nil
|
||||
}
|
||||
|
||||
const bucketsRoot = "/buckets"
|
||||
|
||||
func bucketBase(bucket string) string {
|
||||
return bucketsRoot + "/" + bucket
|
||||
}
|
||||
|
||||
// bucketTableFor returns the bucket whose table alone holds everything below
|
||||
// dirPath, or "" when dirPath is not inside a real bucket (as getTxOrDB routes).
|
||||
func bucketTableFor(dirPath util.FullPath) string {
|
||||
p := string(dirPath)
|
||||
if !strings.HasPrefix(p, bucketsRoot+"/") {
|
||||
return ""
|
||||
}
|
||||
rest := p[len(bucketsRoot)+1:]
|
||||
bucket := rest
|
||||
if i := strings.Index(rest, "/"); i >= 0 {
|
||||
bucket = rest[:i]
|
||||
}
|
||||
if bucket == "" || strings.HasPrefix(bucket, ".") || !isValidBucket(bucket) {
|
||||
return ""
|
||||
}
|
||||
return bucket
|
||||
}
|
||||
|
||||
// bucketNames is the snapshot-listed and cached-only buckets whose tables a
|
||||
// scan may read.
|
||||
type bucketNames struct {
|
||||
names []string
|
||||
seen map[string]bool // listed under /buckets inside the snapshot
|
||||
}
|
||||
|
||||
// listBucketNames reads the directories under /buckets within the snapshot
|
||||
// and keeps those that route to their own table, plus the buckets this
|
||||
// process has already seen (their directory entry may be gone, their rows not).
|
||||
func (store *AbstractSqlStore) listBucketNames(ctx context.Context, tx *sql.Tx, cachedBuckets []string) (bucketNames, error) {
|
||||
var names bucketNames
|
||||
names.seen = make(map[string]bool)
|
||||
added := make(map[string]bool)
|
||||
add := func(name string) {
|
||||
if added[name] || strings.HasPrefix(name, ".") || !isValidBucket(name) {
|
||||
return
|
||||
}
|
||||
added[name] = true
|
||||
names.names = append(names.names, name)
|
||||
}
|
||||
|
||||
const page = 1000
|
||||
startFrom := ""
|
||||
for {
|
||||
type row struct {
|
||||
name string
|
||||
data []byte
|
||||
}
|
||||
var rows []row
|
||||
if err := func() error {
|
||||
res, err := tx.QueryContext(ctx, store.GetSqlListExclusive(DEFAULT_TABLE), util.HashStringToLong(bucketsRoot), startFrom, bucketsRoot, "%", page)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list %s: %v", bucketsRoot, err)
|
||||
}
|
||||
defer res.Close()
|
||||
for res.Next() {
|
||||
var r row
|
||||
if err := res.Scan(&r.name, &r.data); err != nil {
|
||||
return fmt.Errorf("scan %s: %v", bucketsRoot, err)
|
||||
}
|
||||
rows = append(rows, r)
|
||||
}
|
||||
return res.Err()
|
||||
}(); err != nil {
|
||||
return bucketNames{}, err
|
||||
}
|
||||
for _, r := range rows {
|
||||
startFrom = r.name
|
||||
entry := &filer.Entry{FullPath: util.NewFullPath(bucketsRoot, r.name)}
|
||||
if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(r.data)); err != nil {
|
||||
return bucketNames{}, fmt.Errorf("scan decode %s: %v", entry.FullPath, err)
|
||||
}
|
||||
if entry.IsDirectory() {
|
||||
names.seen[r.name] = true
|
||||
add(r.name)
|
||||
}
|
||||
}
|
||||
if len(rows) < page {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range cachedBuckets {
|
||||
add(name)
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// scanOneTable reads one table in primary-key pages and hands the entries
|
||||
// strictly below dirPath to fn (the root itself is read separately). Key-value
|
||||
// records share the default table; GenDirAndName stores the raw key bytes as
|
||||
// their dirhash while entry rows hash the directory, so the mismatch
|
||||
// identifies them exactly.
|
||||
func (store *AbstractSqlStore) scanOneTable(ctx context.Context, tx *sql.Tx, t scanTable, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error {
|
||||
type row struct {
|
||||
dirHash int64
|
||||
name, dir string
|
||||
data []byte
|
||||
}
|
||||
cursorHash, cursorName := int64(math.MinInt64), ""
|
||||
firstPage := true
|
||||
for {
|
||||
var page []row
|
||||
if t.cachedOnly && firstPage {
|
||||
// A missing table can only be met by the first page read: once a
|
||||
// row came back the tx holds the table against a drop. On
|
||||
// PostgreSQL the failed query aborts the transaction, so the skip
|
||||
// needs a savepoint to roll back to before the next table's read.
|
||||
if _, err := tx.ExecContext(ctx, "SAVEPOINT seaweed_scan_bucket"); err != nil {
|
||||
return fmt.Errorf("scan table %s: %v", t.table, err)
|
||||
}
|
||||
}
|
||||
sqlText, args := store.GetSqlScanPage(t.table, cursorHash, cursorName, scanPageSize)
|
||||
rows, err := tx.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
if t.cachedOnly && isMissingTableError(err) {
|
||||
tx.ExecContext(ctx, "ROLLBACK TO SAVEPOINT seaweed_scan_bucket")
|
||||
return nil // a cached-only name whose table was dropped meanwhile
|
||||
}
|
||||
return missingTableOr(err, t.table, fmt.Sprintf("scan table %s", t.table))
|
||||
}
|
||||
firstPage = false
|
||||
if err := func() error {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var r row
|
||||
if err := rows.Scan(&r.dirHash, &r.name, &r.dir, &r.data); err != nil {
|
||||
return fmt.Errorf("scan table %s: %v", t.table, err)
|
||||
}
|
||||
page = append(page, r)
|
||||
}
|
||||
return rows.Err()
|
||||
}(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, r := range page {
|
||||
cursorHash, cursorName = r.dirHash, r.name
|
||||
if r.dirHash != util.HashStringToLong(r.dir) {
|
||||
continue // a key-value record, see GenDirAndName
|
||||
}
|
||||
fullPath := util.NewFullPath(t.base+r.dir, r.name)
|
||||
if !isStrictlyBelow(fullPath, dirPath) {
|
||||
continue
|
||||
}
|
||||
entry := &filer.Entry{FullPath: fullPath}
|
||||
if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(r.data)); err != nil {
|
||||
return fmt.Errorf("scan decode %s: %v", fullPath, err)
|
||||
}
|
||||
if err := fn(ctx, entry); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(page) < scanPageSize {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// missingTableOr names the condition when a bucket's table is not there. The
|
||||
// scan fails closed: from inside the snapshot a table that was never created
|
||||
// cannot be told from one dropped a moment ago, whose rows the snapshot still
|
||||
// counts, and treating either as empty would let fsck purge referenced chunks.
|
||||
func missingTableOr(err error, table, what string) error {
|
||||
if table != DEFAULT_TABLE && isMissingTableError(err) {
|
||||
return fmt.Errorf("bucket %s has a directory entry but no table (never created, or dropped during the scan): %v", table, err)
|
||||
}
|
||||
return fmt.Errorf("%s: %v", what, err)
|
||||
}
|
||||
|
||||
// isMissingTableError recognizes a missing table by message, as InsertEntry
|
||||
// recognizes duplicates: PostgreSQL `relation "x" does not exist` (SQLSTATE
|
||||
// 42P01), MySQL error 1146 (SQLSTATE 42S02), SQLite "no such table".
|
||||
func isMissingTableError(err error) bool {
|
||||
msg := strings.ToLower(err.Error())
|
||||
return strings.Contains(msg, "42p01") ||
|
||||
(strings.Contains(msg, "relation \"") && strings.Contains(msg, "does not exist")) ||
|
||||
strings.Contains(msg, "42s02") ||
|
||||
strings.Contains(msg, "error 1146") ||
|
||||
strings.Contains(msg, "no such table")
|
||||
}
|
||||
|
||||
// isStrictlyBelow reports whether p is a descendant of dir (not dir itself).
|
||||
func isStrictlyBelow(p, dir util.FullPath) bool {
|
||||
if dir == "/" {
|
||||
return p != "/"
|
||||
}
|
||||
return strings.HasPrefix(string(p), string(dir)+"/")
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package abstract_sql
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func TestBucketTableFor(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"/": "",
|
||||
"/buckets": "",
|
||||
"/buckets/": "",
|
||||
"/buckets/mybucket": "mybucket",
|
||||
"/buckets/mybucket/a/b": "mybucket",
|
||||
"/buckets/.system/owners": "",
|
||||
"/buckets/Invalid_Bucket/x": "",
|
||||
"/topics/.system/log": "",
|
||||
"/bucketsfoo/mybucket": "",
|
||||
}
|
||||
for path, want := range cases {
|
||||
if got := bucketTableFor(util.FullPath(path)); got != want {
|
||||
t.Errorf("bucketTableFor(%s) = %q, want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only a missing table may be skipped; any other schema error must fail the
|
||||
// scan, or a bucket's live chunks would be reported as orphans.
|
||||
func TestIsMissingTableError(t *testing.T) {
|
||||
for _, msg := range []string{
|
||||
`ERROR: relation "ghost" does not exist (SQLSTATE 42P01)`,
|
||||
`Error 1146 (42S02): Table 'seaweedfs.ghost' doesn't exist`,
|
||||
`SQL logic error: no such table: ghost (1)`,
|
||||
} {
|
||||
if !isMissingTableError(errors.New(msg)) {
|
||||
t.Errorf("not recognized as a missing table: %s", msg)
|
||||
}
|
||||
}
|
||||
for _, msg := range []string{
|
||||
`ERROR: column "directory" does not exist (SQLSTATE 42703)`,
|
||||
`Error 1054 (42S22): Unknown column 'directory' in 'field list'`,
|
||||
`SQL logic error: no such column: directory (1)`,
|
||||
"connection refused",
|
||||
} {
|
||||
if isMissingTableError(errors.New(msg)) {
|
||||
t.Errorf("wrongly taken for a missing table: %s", msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -416,6 +416,28 @@ func (f *Filer) RollbackTransaction(ctx context.Context) error {
|
||||
return f.Store.RollbackTransaction(ctx)
|
||||
}
|
||||
|
||||
type syncWriteKey struct{}
|
||||
|
||||
// WithSyncWrite marks the entry writes under ctx durable-on-ack: a store that
|
||||
// buffers writes by default must flush them before returning.
|
||||
func WithSyncWrite(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, syncWriteKey{}, true)
|
||||
}
|
||||
|
||||
// SyncWrite reports whether ctx carries a durability request for entry writes.
|
||||
func SyncWrite(ctx context.Context) bool {
|
||||
return ctx.Value(syncWriteKey{}) != nil
|
||||
}
|
||||
|
||||
// syncWriteContext flags ctx when the storage rule on p asks for fsync, so the
|
||||
// store write is held to the same durability as the data path.
|
||||
func (f *Filer) syncWriteContext(ctx context.Context, p util.FullPath) context.Context {
|
||||
if f.FilerConf != nil && f.FilerConf.MatchStorageRule(string(p)).Fsync {
|
||||
return WithSyncWrite(ctx)
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
|
||||
// CreateEntry creates or replaces an entry. When existing is non-nil the caller
|
||||
// has already fetched the current entry at this path under a path lock, and it
|
||||
// is reused instead of looking the store up again; pass nil to have CreateEntry
|
||||
@@ -426,6 +448,8 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx = f.syncWriteContext(ctx, entry.FullPath)
|
||||
|
||||
if entry.FullPath.IsLongerFileName(maxFilenameLength) {
|
||||
return filer_pb.ErrEntryNameTooLong
|
||||
}
|
||||
@@ -631,6 +655,7 @@ func (f *Filer) DirectoryAttributes(ctx context.Context, dirPath util.FullPath)
|
||||
// deleted with, so a restore cannot hand back a directory more permissive than the one
|
||||
// it replaces - including when someone else already put it back with wider ones.
|
||||
func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath, attrs empty_folder_cleanup.DirectoryAttributes) error {
|
||||
ctx = f.syncWriteContext(ctx, dirPath)
|
||||
existing, err := f.FindEntry(ctx, dirPath)
|
||||
if err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return err
|
||||
@@ -680,6 +705,7 @@ func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath,
|
||||
}
|
||||
|
||||
func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry, isFromOtherCluster bool) (err error) {
|
||||
ctx = f.syncWriteContext(ctx, entry.FullPath)
|
||||
if oldEntry != nil {
|
||||
entry.Attr.Crtime = oldEntry.Attr.Crtime
|
||||
if oldEntry.Attr.Inode != 0 {
|
||||
|
||||
@@ -122,6 +122,8 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx = f.syncWriteContext(ctx, p)
|
||||
|
||||
entry, findErr := f.FindEntry(ctx, p)
|
||||
if findErr != nil {
|
||||
return findErr
|
||||
|
||||
@@ -44,6 +44,22 @@ type FilerStore interface {
|
||||
Shutdown()
|
||||
}
|
||||
|
||||
// ErrUnsupportedEntryScan is returned by EntryScanner.ScanEntries, before
|
||||
// any entry is visited, when the store cannot serve the scan.
|
||||
var ErrUnsupportedEntryScan = errors.New("entry scan not supported by this filer store")
|
||||
|
||||
// EntryScanner is an optional FilerStore capability: visit every entry at and
|
||||
// below dirPath, in any order, from one consistent snapshot of the store.
|
||||
//
|
||||
// A directory walk sees each directory at a different time, so an entry
|
||||
// renamed from a directory not yet listed into one already listed is never
|
||||
// seen; volume.fsck then purges its chunks as orphans. A snapshot sees it
|
||||
// exactly once. The context handed to fn carries the snapshot, so store reads
|
||||
// made with it (hard-link records) see the same instant.
|
||||
type EntryScanner interface {
|
||||
ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *Entry) error) error
|
||||
}
|
||||
|
||||
type BucketAware interface {
|
||||
OnBucketCreation(bucket string)
|
||||
OnBucketDeletion(bucket string)
|
||||
|
||||
@@ -2,6 +2,7 @@ package filer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
@@ -19,6 +20,7 @@ import (
|
||||
var (
|
||||
_ = VirtualFilerStore(&FilerStoreWrapper{})
|
||||
_ = Debuggable(&FilerStoreWrapper{})
|
||||
_ = EntryScanner(&FilerStoreWrapper{})
|
||||
)
|
||||
|
||||
type VirtualFilerStore interface {
|
||||
@@ -402,6 +404,33 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context,
|
||||
return lastFileName, err
|
||||
}
|
||||
|
||||
// ScanEntries delegates to the default store. With path-specific stores
|
||||
// configured, entries below dirPath could live in several stores and one
|
||||
// snapshot per store is not one snapshot, so the scan is declined. Hard links
|
||||
// are resolved inside the snapshot; a store error fails the scan (the per-path
|
||||
// copy can be stale), a missing record falls back to it as listings do.
|
||||
func (fsw *FilerStoreWrapper) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *Entry) error) error {
|
||||
if fsw.hasPathSpecificStore {
|
||||
return ErrUnsupportedEntryScan
|
||||
}
|
||||
scanner, ok := fsw.defaultStore.(EntryScanner)
|
||||
if !ok {
|
||||
return ErrUnsupportedEntryScan
|
||||
}
|
||||
stats.FilerStoreCounter.WithLabelValues(fsw.defaultStore.GetName(), "scan").Inc()
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
stats.FilerStoreHistogram.WithLabelValues(fsw.defaultStore.GetName(), "scan").Observe(time.Since(start).Seconds())
|
||||
}()
|
||||
return scanner.ScanEntries(ctx, dirPath, func(scanCtx context.Context, entry *Entry) error {
|
||||
if err := fsw.maybeReadHardLink(scanCtx, entry); err != nil && !errors.Is(err, ErrKvNotFound) {
|
||||
return fmt.Errorf("resolve hard link of %s: %w", entry.FullPath, err)
|
||||
}
|
||||
filer_pb.AfterEntryDeserialization(entry.GetChunks())
|
||||
return fn(scanCtx, entry)
|
||||
})
|
||||
}
|
||||
|
||||
func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
|
||||
actualStore := fsw.getActualStore(dirPath + "/")
|
||||
|
||||
|
||||
@@ -79,6 +79,15 @@ func (store *LevelDBStore) RollbackTransaction(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var syncWriteOptions = &opt.WriteOptions{Sync: true}
|
||||
|
||||
func writeOptions(ctx context.Context) *opt.WriteOptions {
|
||||
if filer.SyncWrite(ctx) {
|
||||
return syncWriteOptions
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
||||
key := genKey(entry.DirAndName())
|
||||
|
||||
@@ -91,7 +100,7 @@ func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer.Entry)
|
||||
value = weed_util.MaybeGzipData(value)
|
||||
}
|
||||
|
||||
err = store.db.Put(key, value, nil)
|
||||
err = store.db.Put(key, value, writeOptions(ctx))
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
|
||||
@@ -168,7 +177,7 @@ func (store *LevelDBStore) FindEntry(ctx context.Context, fullpath weed_util.Ful
|
||||
func (store *LevelDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
|
||||
key := genKey(fullpath.DirAndName())
|
||||
|
||||
err = store.db.Delete(key, nil)
|
||||
err = store.db.Delete(key, writeOptions(ctx))
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete %s : %v", fullpath, err)
|
||||
}
|
||||
@@ -195,7 +204,7 @@ func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath we
|
||||
}
|
||||
iter.Release()
|
||||
|
||||
err = store.db.Write(batch, nil)
|
||||
err = store.db.Write(batch, writeOptions(ctx))
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete %s : %v", fullpath, err)
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
@@ -28,6 +29,11 @@ type LevelDB2Store struct {
|
||||
dbs []*leveldb.DB
|
||||
dbCount int
|
||||
ReadOnly bool
|
||||
|
||||
blockCacheSizeMB int
|
||||
writeBufferSizeMB int
|
||||
blockSizeKB int
|
||||
compactionTableSizeMB int
|
||||
}
|
||||
|
||||
func (store *LevelDB2Store) GetName() string {
|
||||
@@ -36,9 +42,54 @@ func (store *LevelDB2Store) GetName() string {
|
||||
|
||||
func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
|
||||
dir := configuration.GetString(prefix + "dir")
|
||||
store.blockCacheSizeMB = configuration.GetInt(prefix + "block_cache_size_mb")
|
||||
store.writeBufferSizeMB = configuration.GetInt(prefix + "write_buffer_size_mb")
|
||||
store.blockSizeKB = configuration.GetInt(prefix + "block_size_kb")
|
||||
store.compactionTableSizeMB = configuration.GetInt(prefix + "compaction_table_size_mb")
|
||||
if err := store.validateSizeOptions(); err != nil {
|
||||
return err
|
||||
}
|
||||
return store.initialize(dir, 8)
|
||||
}
|
||||
|
||||
// validateSizeOptions rejects sizes too large to convert to bytes without
|
||||
// overflowing int on this platform.
|
||||
func (store *LevelDB2Store) validateSizeOptions() error {
|
||||
for name, kv := range map[string][2]int{
|
||||
"block_cache_size_mb": {store.blockCacheSizeMB, 1 << 20},
|
||||
"write_buffer_size_mb": {store.writeBufferSizeMB, 1 << 20},
|
||||
"block_size_kb": {store.blockSizeKB, 1 << 10},
|
||||
"compaction_table_size_mb": {store.compactionTableSizeMB, 1 << 20},
|
||||
} {
|
||||
if kv[0] > math.MaxInt/kv[1] {
|
||||
return fmt.Errorf("leveldb2 option %s=%d overflows int", name, kv[0])
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *LevelDB2Store) buildOptions() *opt.Options {
|
||||
opts := &opt.Options{
|
||||
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
|
||||
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
|
||||
Filter: filter.NewBloomFilter(8), // false positive rate 0.02
|
||||
ReadOnly: store.ReadOnly,
|
||||
}
|
||||
if store.blockCacheSizeMB > 0 {
|
||||
opts.BlockCacheCapacity = store.blockCacheSizeMB * 1024 * 1024
|
||||
}
|
||||
if store.writeBufferSizeMB > 0 {
|
||||
opts.WriteBuffer = store.writeBufferSizeMB * 1024 * 1024
|
||||
}
|
||||
if store.blockSizeKB > 0 {
|
||||
opts.BlockSize = store.blockSizeKB * 1024
|
||||
}
|
||||
if store.compactionTableSizeMB > 0 {
|
||||
opts.CompactionTableSize = store.compactionTableSizeMB * 1024 * 1024
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
|
||||
glog.Infof("filer store leveldb2 dir: %s", dir)
|
||||
os.MkdirAll(dir, 0755)
|
||||
@@ -46,12 +97,7 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
|
||||
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
|
||||
}
|
||||
|
||||
opts := &opt.Options{
|
||||
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
|
||||
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
|
||||
Filter: filter.NewBloomFilter(8), // false positive rate 0.02
|
||||
ReadOnly: store.ReadOnly,
|
||||
}
|
||||
opts := store.buildOptions()
|
||||
|
||||
for d := 0; d < dbCount; d++ {
|
||||
dbFolder := fmt.Sprintf("%s/%02d", dir, d)
|
||||
@@ -81,6 +127,15 @@ func (store *LevelDB2Store) RollbackTransaction(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var syncWriteOptions = &opt.WriteOptions{Sync: true}
|
||||
|
||||
func writeOptions(ctx context.Context) *opt.WriteOptions {
|
||||
if filer.SyncWrite(ctx) {
|
||||
return syncWriteOptions
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
||||
dir, name := entry.DirAndName()
|
||||
key, partitionId := genKey(dir, name, store.dbCount)
|
||||
@@ -94,7 +149,7 @@ func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer.Entry)
|
||||
value = weed_util.MaybeGzipData(value)
|
||||
}
|
||||
|
||||
err = store.dbs[partitionId].Put(key, value, nil)
|
||||
err = store.dbs[partitionId].Put(key, value, writeOptions(ctx))
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
|
||||
@@ -140,7 +195,7 @@ func (store *LevelDB2Store) DeleteEntry(ctx context.Context, fullpath weed_util.
|
||||
dir, name := fullpath.DirAndName()
|
||||
key, partitionId := genKey(dir, name, store.dbCount)
|
||||
|
||||
err = store.dbs[partitionId].Delete(key, nil)
|
||||
err = store.dbs[partitionId].Delete(key, writeOptions(ctx))
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete %s : %v", fullpath, err)
|
||||
}
|
||||
@@ -167,7 +222,7 @@ func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath w
|
||||
}
|
||||
iter.Release()
|
||||
|
||||
err = store.dbs[partitionId].Write(batch, nil)
|
||||
err = store.dbs[partitionId].Write(batch, writeOptions(ctx))
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete %s : %v", fullpath, err)
|
||||
|
||||
@@ -2,6 +2,8 @@ package leveldb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
@@ -62,6 +64,93 @@ func TestCreateAndFind(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
type fakeConfiguration map[string]interface{}
|
||||
|
||||
func (c fakeConfiguration) GetString(key string) string {
|
||||
value, _ := c[key].(string)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) GetBool(key string) bool {
|
||||
value, _ := c[key].(bool)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) GetInt(key string) int {
|
||||
value, _ := c[key].(int)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) GetStringSlice(key string) []string {
|
||||
value, _ := c[key].([]string)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) SetDefault(key string, value interface{}) {
|
||||
if _, found := c[key]; !found {
|
||||
c[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeOptions(t *testing.T) {
|
||||
store := &LevelDB2Store{}
|
||||
if err := store.Initialize(fakeConfiguration{
|
||||
"leveldb2.dir": t.TempDir(),
|
||||
"leveldb2.block_cache_size_mb": 64,
|
||||
"leveldb2.write_buffer_size_mb": 32,
|
||||
"leveldb2.block_size_kb": 16,
|
||||
"leveldb2.compaction_table_size_mb": 8,
|
||||
}, "leveldb2."); err != nil {
|
||||
t.Fatalf("initialize: %v", err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
|
||||
opts := store.buildOptions()
|
||||
if opts.BlockCacheCapacity != 64*1024*1024 {
|
||||
t.Errorf("block cache %d, want 64MiB", opts.BlockCacheCapacity)
|
||||
}
|
||||
if opts.WriteBuffer != 32*1024*1024 {
|
||||
t.Errorf("write buffer %d, want 32MiB", opts.WriteBuffer)
|
||||
}
|
||||
if opts.BlockSize != 16*1024 {
|
||||
t.Errorf("block size %d, want 16KiB", opts.BlockSize)
|
||||
}
|
||||
if opts.CompactionTableSize != 8*1024*1024 {
|
||||
t.Errorf("compaction table size %d, want 8MiB", opts.CompactionTableSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeDefaultOptions(t *testing.T) {
|
||||
store := &LevelDB2Store{}
|
||||
if err := store.Initialize(fakeConfiguration{
|
||||
"leveldb2.dir": t.TempDir(),
|
||||
"leveldb2.block_cache_size_mb": -1,
|
||||
"leveldb2.write_buffer_size_mb": 0,
|
||||
}, "leveldb2."); err != nil {
|
||||
t.Fatalf("initialize: %v", err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
|
||||
opts := store.buildOptions()
|
||||
if opts.BlockCacheCapacity != 32*1024*1024 {
|
||||
t.Errorf("block cache %d, want the 32MiB default", opts.BlockCacheCapacity)
|
||||
}
|
||||
if opts.WriteBuffer != 16*1024*1024 {
|
||||
t.Errorf("write buffer %d, want the 16MiB default", opts.WriteBuffer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeRejectsOversizedOption(t *testing.T) {
|
||||
store := &LevelDB2Store{}
|
||||
err := store.Initialize(fakeConfiguration{
|
||||
"leveldb2.dir": t.TempDir(),
|
||||
"leveldb2.write_buffer_size_mb": math.MaxInt/(1<<20) + 1,
|
||||
}, "leveldb2.")
|
||||
if err == nil || !strings.Contains(err.Error(), "write_buffer_size_mb") {
|
||||
t.Fatalf("expected overflow rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyRoot(t *testing.T) {
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
|
||||
dir := t.TempDir()
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -35,6 +36,11 @@ type LevelDB3Store struct {
|
||||
dbs map[string]*leveldb.DB
|
||||
dbsLock sync.RWMutex
|
||||
ReadOnly bool
|
||||
|
||||
blockCacheSizeMB int
|
||||
writeBufferSizeMB int
|
||||
blockSizeKB int
|
||||
compactionTableSizeMB int
|
||||
}
|
||||
|
||||
func (store *LevelDB3Store) GetName() string {
|
||||
@@ -43,9 +49,58 @@ func (store *LevelDB3Store) GetName() string {
|
||||
|
||||
func (store *LevelDB3Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
|
||||
dir := configuration.GetString(prefix + "dir")
|
||||
store.blockCacheSizeMB = configuration.GetInt(prefix + "block_cache_size_mb")
|
||||
store.writeBufferSizeMB = configuration.GetInt(prefix + "write_buffer_size_mb")
|
||||
store.blockSizeKB = configuration.GetInt(prefix + "block_size_kb")
|
||||
store.compactionTableSizeMB = configuration.GetInt(prefix + "compaction_table_size_mb")
|
||||
if err := store.validateSizeOptions(); err != nil {
|
||||
return err
|
||||
}
|
||||
return store.initialize(dir)
|
||||
}
|
||||
|
||||
// validateSizeOptions rejects sizes too large to convert to bytes without
|
||||
// overflowing int on this platform.
|
||||
func (store *LevelDB3Store) validateSizeOptions() error {
|
||||
for name, kv := range map[string][2]int{
|
||||
"block_cache_size_mb": {store.blockCacheSizeMB, 1 << 20},
|
||||
"write_buffer_size_mb": {store.writeBufferSizeMB, 1 << 20},
|
||||
"block_size_kb": {store.blockSizeKB, 1 << 10},
|
||||
"compaction_table_size_mb": {store.compactionTableSizeMB, 1 << 20},
|
||||
} {
|
||||
if kv[0] > math.MaxInt/kv[1] {
|
||||
return fmt.Errorf("leveldb3 option %s=%d overflows int", name, kv[0])
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *LevelDB3Store) buildOptions(name string) *opt.Options {
|
||||
opts := &opt.Options{
|
||||
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
|
||||
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
|
||||
Filter: filter.NewBloomFilter(8),
|
||||
ReadOnly: store.ReadOnly,
|
||||
}
|
||||
if name != DEFAULT {
|
||||
opts.BlockCacheCapacity = 16 * 1024 * 1024 // default value is 8MiB
|
||||
opts.WriteBuffer = 8 * 1024 * 1024 // default value is 4MiB
|
||||
}
|
||||
if store.blockCacheSizeMB > 0 {
|
||||
opts.BlockCacheCapacity = store.blockCacheSizeMB * 1024 * 1024
|
||||
}
|
||||
if store.writeBufferSizeMB > 0 {
|
||||
opts.WriteBuffer = store.writeBufferSizeMB * 1024 * 1024
|
||||
}
|
||||
if store.blockSizeKB > 0 {
|
||||
opts.BlockSize = store.blockSizeKB * 1024
|
||||
}
|
||||
if store.compactionTableSizeMB > 0 {
|
||||
opts.CompactionTableSize = store.compactionTableSizeMB * 1024 * 1024
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
func (store *LevelDB3Store) initialize(dir string) (err error) {
|
||||
glog.Infof("filer store leveldb3 dir: %s", dir)
|
||||
os.MkdirAll(dir, 0755)
|
||||
@@ -65,21 +120,7 @@ func (store *LevelDB3Store) initialize(dir string) (err error) {
|
||||
}
|
||||
|
||||
func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) {
|
||||
bloom := filter.NewBloomFilter(8) // false positive rate 0.02
|
||||
opts := &opt.Options{
|
||||
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
|
||||
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
|
||||
Filter: bloom,
|
||||
ReadOnly: store.ReadOnly,
|
||||
}
|
||||
if name != DEFAULT {
|
||||
opts = &opt.Options{
|
||||
BlockCacheCapacity: 16 * 1024 * 1024, // default value is 8MiB
|
||||
WriteBuffer: 8 * 1024 * 1024, // default value is 4MiB
|
||||
Filter: bloom,
|
||||
ReadOnly: store.ReadOnly,
|
||||
}
|
||||
}
|
||||
opts := store.buildOptions(name)
|
||||
|
||||
dbFolder := fmt.Sprintf("%s/%s", store.dir, name)
|
||||
os.MkdirAll(dbFolder, 0755)
|
||||
@@ -180,6 +221,15 @@ func (store *LevelDB3Store) RollbackTransaction(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var syncWriteOptions = &opt.WriteOptions{Sync: true}
|
||||
|
||||
func writeOptions(ctx context.Context) *opt.WriteOptions {
|
||||
if filer.SyncWrite(ctx) {
|
||||
return syncWriteOptions
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
||||
|
||||
db, _, shortPath, err := store.findDB(entry.FullPath, false)
|
||||
@@ -199,7 +249,7 @@ func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry)
|
||||
value = weed_util.MaybeGzipData(value)
|
||||
}
|
||||
|
||||
err = db.Put(key, value, nil)
|
||||
err = db.Put(key, value, writeOptions(ctx))
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
|
||||
@@ -257,7 +307,7 @@ func (store *LevelDB3Store) DeleteEntry(ctx context.Context, fullpath weed_util.
|
||||
dir, name := shortPath.DirAndName()
|
||||
key := genKey(dir, name)
|
||||
|
||||
err = db.Delete(key, nil)
|
||||
err = db.Delete(key, writeOptions(ctx))
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete %s : %v", fullpath, err)
|
||||
}
|
||||
@@ -298,7 +348,7 @@ func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath w
|
||||
}
|
||||
iter.Release()
|
||||
|
||||
err = db.Write(batch, nil)
|
||||
err = db.Write(batch, writeOptions(ctx))
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete %s : %v", fullpath, err)
|
||||
|
||||
@@ -2,6 +2,8 @@ package leveldb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
@@ -62,6 +64,103 @@ func TestCreateAndFind(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
type fakeConfiguration map[string]interface{}
|
||||
|
||||
func (c fakeConfiguration) GetString(key string) string {
|
||||
value, _ := c[key].(string)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) GetBool(key string) bool {
|
||||
value, _ := c[key].(bool)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) GetInt(key string) int {
|
||||
value, _ := c[key].(int)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) GetStringSlice(key string) []string {
|
||||
value, _ := c[key].([]string)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c fakeConfiguration) SetDefault(key string, value interface{}) {
|
||||
if _, found := c[key]; !found {
|
||||
c[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeOptions(t *testing.T) {
|
||||
store := &LevelDB3Store{}
|
||||
if err := store.Initialize(fakeConfiguration{
|
||||
"leveldb3.dir": t.TempDir(),
|
||||
"leveldb3.block_cache_size_mb": 64,
|
||||
"leveldb3.write_buffer_size_mb": 32,
|
||||
"leveldb3.block_size_kb": 16,
|
||||
"leveldb3.compaction_table_size_mb": 8,
|
||||
}, "leveldb3."); err != nil {
|
||||
t.Fatalf("initialize: %v", err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
|
||||
for _, name := range []string{DEFAULT, "bucket-a"} {
|
||||
opts := store.buildOptions(name)
|
||||
if opts.BlockCacheCapacity != 64*1024*1024 {
|
||||
t.Errorf("%s block cache %d, want 64MiB", name, opts.BlockCacheCapacity)
|
||||
}
|
||||
if opts.WriteBuffer != 32*1024*1024 {
|
||||
t.Errorf("%s write buffer %d, want 32MiB", name, opts.WriteBuffer)
|
||||
}
|
||||
if opts.BlockSize != 16*1024 {
|
||||
t.Errorf("%s block size %d, want 16KiB", name, opts.BlockSize)
|
||||
}
|
||||
if opts.CompactionTableSize != 8*1024*1024 {
|
||||
t.Errorf("%s compaction table size %d, want 8MiB", name, opts.CompactionTableSize)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeDefaultOptions(t *testing.T) {
|
||||
store := &LevelDB3Store{}
|
||||
if err := store.Initialize(fakeConfiguration{
|
||||
"leveldb3.dir": t.TempDir(),
|
||||
"leveldb3.block_cache_size_mb": -1,
|
||||
"leveldb3.write_buffer_size_mb": 0,
|
||||
}, "leveldb3."); err != nil {
|
||||
t.Fatalf("initialize: %v", err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
|
||||
opts := store.buildOptions(DEFAULT)
|
||||
if opts.BlockCacheCapacity != 32*1024*1024 {
|
||||
t.Errorf("block cache %d, want the 32MiB default", opts.BlockCacheCapacity)
|
||||
}
|
||||
if opts.WriteBuffer != 16*1024*1024 {
|
||||
t.Errorf("write buffer %d, want the 16MiB default", opts.WriteBuffer)
|
||||
}
|
||||
|
||||
bucketOpts := store.buildOptions("bucket-a")
|
||||
if bucketOpts.BlockCacheCapacity != 16*1024*1024 {
|
||||
t.Errorf("bucket block cache %d, want the 16MiB default", bucketOpts.BlockCacheCapacity)
|
||||
}
|
||||
if bucketOpts.WriteBuffer != 8*1024*1024 {
|
||||
t.Errorf("bucket write buffer %d, want the 8MiB default", bucketOpts.WriteBuffer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeRejectsOversizedOption(t *testing.T) {
|
||||
store := &LevelDB3Store{}
|
||||
err := store.Initialize(fakeConfiguration{
|
||||
"leveldb3.dir": t.TempDir(),
|
||||
"leveldb3.write_buffer_size_mb": math.MaxInt/(1<<20) + 1,
|
||||
}, "leveldb3.")
|
||||
if err == nil || !strings.Contains(err.Error(), "write_buffer_size_mb") {
|
||||
t.Fatalf("expected overflow rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyRoot(t *testing.T) {
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
|
||||
dir := t.TempDir()
|
||||
|
||||
@@ -62,6 +62,15 @@ func (gen *SqlGenMysql) nameExpr() string {
|
||||
return "`name`"
|
||||
}
|
||||
|
||||
// GetSqlScanPage spells the cursor as two primary-key ranges: MySQL 5.7 does
|
||||
// not turn a row-value comparison into a range seek. The name keeps the
|
||||
// column's own collation (BINARY would filesort every page); the scan promises
|
||||
// no lexical order. SQLite overrides this method.
|
||||
func (gen *SqlGenMysql) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) {
|
||||
return fmt.Sprintf("SELECT `dirhash`, `name`, `directory`, `meta` FROM `%s` WHERE `dirhash` > ? OR (`dirhash` = ? AND `name` > ?) ORDER BY `dirhash` ASC, `name` ASC LIMIT ?", tableName),
|
||||
[]interface{}{dirhash, dirhash, name, limit}
|
||||
}
|
||||
|
||||
func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string {
|
||||
name := gen.nameExpr()
|
||||
return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s > ? AND `directory` = ? AND %s LIKE ? ESCAPE '!' ORDER BY %s ASC LIMIT ?", tableName, name, name, name)
|
||||
|
||||
@@ -75,6 +75,14 @@ func (gen *SqlGenPostgres) nameExpr() string {
|
||||
return "name"
|
||||
}
|
||||
|
||||
// GetSqlScanPage uses a row-value comparison, one primary-key range seek on
|
||||
// PostgreSQL. The name keeps the column's own collation (COLLATE "C" would
|
||||
// sort every page); the scan promises no lexical order.
|
||||
func (gen *SqlGenPostgres) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) {
|
||||
return fmt.Sprintf(`SELECT dirhash, name, directory, meta FROM "%s" WHERE (dirhash, name) > ($1, $2) ORDER BY dirhash ASC, name ASC LIMIT $3`, tableName),
|
||||
[]interface{}{dirhash, name, limit}
|
||||
}
|
||||
|
||||
func (gen *SqlGenPostgres) GetSqlListExclusive(tableName string) string {
|
||||
name := gen.nameExpr()
|
||||
return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>$2 AND directory=$3 AND %s like $4 ESCAPE '!' ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)
|
||||
|
||||
@@ -30,6 +30,31 @@ func (store *SqliteStore) GetName() string {
|
||||
return "sqlite"
|
||||
}
|
||||
|
||||
// sqliteSqlGen is the MySQL generator with SQLite's own page cursor: SQLite
|
||||
// seeks a row-value comparison on the primary key but scans the OR form.
|
||||
type sqliteSqlGen struct {
|
||||
*mysql.SqlGenMysql
|
||||
}
|
||||
|
||||
func (gen sqliteSqlGen) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) {
|
||||
return fmt.Sprintf("SELECT `dirhash`, `name`, `directory`, `meta` FROM `%s` WHERE (`dirhash`, `name`) > (?, ?) ORDER BY `dirhash` ASC, `name` ASC LIMIT ?", tableName),
|
||||
[]interface{}{dirhash, name, limit}
|
||||
}
|
||||
|
||||
// ScanEntries offers the snapshot scan only in WAL mode: with a rollback
|
||||
// journal a long read transaction holds a shared lock and writers fail after
|
||||
// the busy timeout.
|
||||
func (store *SqliteStore) ScanEntries(ctx context.Context, dirPath util.FullPath, fn func(ctx context.Context, entry *filer.Entry) error) error {
|
||||
var mode string
|
||||
if err := store.DB.QueryRowContext(ctx, "PRAGMA journal_mode").Scan(&mode); err != nil {
|
||||
return fmt.Errorf("%w: journal_mode: %v", filer.ErrUnsupportedEntryScan, err)
|
||||
}
|
||||
if !strings.EqualFold(mode, "wal") {
|
||||
return fmt.Errorf("%w: sqlite journal_mode is %q, the scan needs wal", filer.ErrUnsupportedEntryScan, mode)
|
||||
}
|
||||
return store.AbstractSqlStore.ScanEntries(ctx, dirPath, fn)
|
||||
}
|
||||
|
||||
func (store *SqliteStore) Initialize(configuration util.Configuration, prefix string) (err error) {
|
||||
dbFile := configuration.GetString(prefix + "dbFile")
|
||||
createTable := `CREATE TABLE IF NOT EXISTS "%s" (
|
||||
@@ -69,11 +94,11 @@ func sqliteDSN(dbFile string) string {
|
||||
func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (err error) {
|
||||
|
||||
store.SupportBucketTable = true
|
||||
store.SqlGenerator = &mysql.SqlGenMysql{
|
||||
store.SqlGenerator = sqliteSqlGen{&mysql.SqlGenMysql{
|
||||
CreateTableSqlTemplate: createTable,
|
||||
DropTableSqlTemplate: "drop table if exists `%s`",
|
||||
UpsertQueryTemplate: upsertQuery,
|
||||
}
|
||||
}}
|
||||
|
||||
dsn := sqliteDSN(dbFile)
|
||||
|
||||
@@ -94,6 +119,11 @@ func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (e
|
||||
}, 1, 1, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
// The main pool is one connection, which a long scan must not hold.
|
||||
if store.ScanDB, err = sql.Open("sqlite", dsn); err != nil {
|
||||
return fmt.Errorf("open scan connection to %s: %v", dbFile, err)
|
||||
}
|
||||
store.ScanDB.SetMaxOpenConns(1)
|
||||
|
||||
if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {
|
||||
return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
//go:build (linux || darwin || windows) && sqlite
|
||||
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
// A rename committed while a scan is running is seen under exactly one of its
|
||||
// names. The scan reads from one snapshot, so unlike a directory walk it can
|
||||
// neither miss the entry when it moves into a directory already listed, nor
|
||||
// count it twice when it moves into one not listed yet.
|
||||
func TestScanEntriesSeesConcurrentRenameExactlyOnce(t *testing.T) {
|
||||
dbFile := filepath.Join(t.TempDir(), "filer.db")
|
||||
reader := &SqliteStore{}
|
||||
if err := reader.initialize(dbFile, createTableSql, upsertQuerySql); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reader.Shutdown()
|
||||
// WAL lets the writer commit while the scan's read transaction is open.
|
||||
if _, err := reader.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writer := &SqliteStore{}
|
||||
if err := writer.initialize(dbFile, createTableSql, upsertQuerySql); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer writer.Shutdown()
|
||||
|
||||
ctx := context.Background()
|
||||
mk := func(p string) *filer.Entry {
|
||||
return &filer.Entry{FullPath: util.FullPath(p), Attr: filer.Attr{Mode: 0644, Mtime: time.Now(), Crtime: time.Now()}}
|
||||
}
|
||||
for _, p := range []string{"/ckpt/a", "/ckpt/b", "/ckpt/tmp/x"} {
|
||||
if err := reader.InsertEntry(ctx, mk(p)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
defer func(n int) { abstract_sql.SetScanPageSizeForTest(n) }(abstract_sql.SetScanPageSizeForTest(2))
|
||||
|
||||
seen := map[string]int{}
|
||||
renamed := false
|
||||
err := reader.ScanEntries(ctx, "/", func(_ context.Context, e *filer.Entry) error {
|
||||
seen[string(e.FullPath)]++
|
||||
if renamed {
|
||||
return nil
|
||||
}
|
||||
renamed = true
|
||||
// /ckpt/tmp/x -> /ckpt/x, committed by another connection mid-scan.
|
||||
if err := writer.InsertEntry(ctx, mk("/ckpt/x")); err != nil {
|
||||
return err
|
||||
}
|
||||
return writer.DeleteEntry(ctx, util.FullPath("/ckpt/tmp/x"))
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !renamed {
|
||||
t.Fatal("scan yielded nothing")
|
||||
}
|
||||
if seen["/ckpt/a"] != 1 || seen["/ckpt/b"] != 1 {
|
||||
t.Errorf("untouched entries seen %d and %d times, want once each", seen["/ckpt/a"], seen["/ckpt/b"])
|
||||
}
|
||||
if old, new := seen["/ckpt/tmp/x"], seen["/ckpt/x"]; old+new != 1 {
|
||||
t.Errorf("renamed entry seen %d time(s) under the old name and %d under the new, want exactly one in total", old, new)
|
||||
} else if old != 1 {
|
||||
t.Errorf("the rename committed after the snapshot was taken, so the scan must show the old name")
|
||||
}
|
||||
if _, err := reader.FindEntry(ctx, util.FullPath("/ckpt/x")); err != nil {
|
||||
t.Errorf("once the scan is over the new name must be visible: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Rows that are not entries, key-value records and a regular file under
|
||||
// /buckets with a bucket-like name, are stepped over; a scan rooted at a bucket
|
||||
// yields the bucket's own row (default table) and its contents (bucket table).
|
||||
func TestScanEntriesRowsAndBucketTables(t *testing.T) {
|
||||
store := &SqliteStore{}
|
||||
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
// The filer creates bucket tables on bucket creation; the store layer alone does not.
|
||||
if err := store.CreateTable(ctx, "realbucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range []*filer.Entry{
|
||||
{FullPath: "/buckets", Attr: filer.Attr{Mode: 0755 | os.ModeDir, Mtime: now, Crtime: now}},
|
||||
{FullPath: "/buckets/realbucket", Attr: filer.Attr{Mode: 0755 | os.ModeDir, Mtime: now, Crtime: now}},
|
||||
{FullPath: "/buckets/realbucket/f1", Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}},
|
||||
{FullPath: "/buckets/readme", Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}},
|
||||
{FullPath: "/topics/t1", Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now}},
|
||||
} {
|
||||
if err := store.InsertEntry(ctx, e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"filer.store.id", "/looks/like/a/path", "x"} {
|
||||
if err := store.KvPut(ctx, []byte(key), []byte("opaque value, not an Entry")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
collect := func(root string) []string {
|
||||
var got []string
|
||||
if err := store.ScanEntries(ctx, util.FullPath(root), func(_ context.Context, e *filer.Entry) error {
|
||||
got = append(got, string(e.FullPath))
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("scan %s: %v", root, err)
|
||||
}
|
||||
sort.Strings(got)
|
||||
return got
|
||||
}
|
||||
if got, want := collect("/"), []string{"/buckets", "/buckets/readme", "/buckets/realbucket", "/buckets/realbucket/f1", "/topics/t1"}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("scan /: got %v, want %v", got, want)
|
||||
}
|
||||
if got, want := collect("/buckets/realbucket"), []string{"/buckets/realbucket", "/buckets/realbucket/f1"}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("scan /buckets/realbucket: got %v, want %v", got, want)
|
||||
}
|
||||
|
||||
// A bucket directory whose table does not exist fails the scan: from the
|
||||
// snapshot it cannot be told from a table dropped a moment ago.
|
||||
if err := store.InsertEntry(ctx, &filer.Entry{FullPath: "/buckets/ghostbucket", Attr: filer.Attr{Mode: 0755 | os.ModeDir, Mtime: now, Crtime: now}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := store.ScanEntries(ctx, "/", func(context.Context, *filer.Entry) error { return nil })
|
||||
if err == nil || !strings.Contains(err.Error(), "ghostbucket") {
|
||||
t.Fatalf("scan over a bucket without a table must fail naming it, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Without WAL the store declines the scan rather than block writers for its
|
||||
// whole duration.
|
||||
func TestScanEntriesDeclinedWithoutWAL(t *testing.T) {
|
||||
store := &SqliteStore{}
|
||||
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
err := store.ScanEntries(context.Background(), "/", func(context.Context, *filer.Entry) error { return nil })
|
||||
if !errors.Is(err, filer.ErrUnsupportedEntryScan) {
|
||||
t.Fatalf("err = %v, want ErrUnsupportedEntryScan", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A store whose pool is one connection, and no scan connection of its own,
|
||||
// declines rather than hold that connection for the whole scan.
|
||||
func TestScanEntriesDeclinedOnSingleConnectionPool(t *testing.T) {
|
||||
store := &SqliteStore{}
|
||||
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store.ScanDB.Close()
|
||||
store.ScanDB = nil
|
||||
err := store.ScanEntries(context.Background(), "/", func(context.Context, *filer.Entry) error { return nil })
|
||||
if !errors.Is(err, filer.ErrUnsupportedEntryScan) {
|
||||
t.Fatalf("err = %v, want ErrUnsupportedEntryScan", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A hard-linked entry's chunks live in the shared key-value record, not in
|
||||
// its path row; the wrapper resolves them from inside the snapshot, on the
|
||||
// single connection the scan holds.
|
||||
func TestScanEntriesResolvesHardLinks(t *testing.T) {
|
||||
store := &SqliteStore{}
|
||||
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wrapper := filer.NewFilerStoreWrapper(store)
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
linked := &filer.Entry{
|
||||
FullPath: "/buckets/hlbucket/f0001",
|
||||
Attr: filer.Attr{Mode: 0644, Mtime: now, Crtime: now},
|
||||
HardLinkId: filer.HardLinkId("hardlink-00000001"),
|
||||
HardLinkCounter: 2,
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 4096}},
|
||||
}
|
||||
if err := wrapper.InsertEntry(ctx, linked); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var got *filer.Entry
|
||||
if err := wrapper.ScanEntries(ctx, "/", func(_ context.Context, e *filer.Entry) error {
|
||||
if e.FullPath == linked.FullPath {
|
||||
got = e
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got == nil || len(got.GetChunks()) != 1 || got.GetChunks()[0].FileId != "3,01637037d6" || got.HardLinkCounter != 2 {
|
||||
t.Fatalf("hard-linked entry not resolved from its key-value record: %+v", got)
|
||||
}
|
||||
|
||||
// Once the record is gone the path row is all there is; the scan hands it
|
||||
// over instead of failing on that entry every time it runs.
|
||||
if err := store.KvDelete(ctx, []byte(linked.HardLinkId)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got = nil
|
||||
if err := wrapper.ScanEntries(ctx, "/", func(_ context.Context, e *filer.Entry) error {
|
||||
if e.FullPath == linked.FullPath {
|
||||
got = e
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("a missing hard-link record must not fail the scan: %v", err)
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatal("entry with a missing hard-link record not handed over")
|
||||
}
|
||||
}
|
||||
|
||||
// The keyset page must be a primary-key seek; an OR-shaped cursor predicate
|
||||
// makes SQLite scan the table from the start for every page.
|
||||
func TestScanPageQuerySeeksPrimaryKey(t *testing.T) {
|
||||
store := &SqliteStore{}
|
||||
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
sqlText, args := store.GetSqlScanPage("filemeta", int64(-1), "", 10)
|
||||
rows, err := store.DB.Query("EXPLAIN QUERY PLAN "+sqlText, args...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var plan []string
|
||||
for rows.Next() {
|
||||
var id, parent, notused int
|
||||
var detail string
|
||||
if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan = append(plan, detail)
|
||||
}
|
||||
joined := strings.Join(plan, " | ")
|
||||
if !strings.Contains(joined, "SEARCH") || strings.Contains(joined, "SCAN") {
|
||||
t.Fatalf("page query does not seek the primary key: %s", joined)
|
||||
}
|
||||
}
|
||||
|
||||
// A bucket name only the process cache remembers may have its table dropped
|
||||
// meanwhile; the scan skips it instead of failing.
|
||||
func TestScanEntriesSkipsCachedOnlyBucketTable(t *testing.T) {
|
||||
store := &SqliteStore{}
|
||||
if err := store.initialize(filepath.Join(t.TempDir(), "filer.db"), createTableSql, upsertQuerySql); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
if _, err := store.DB.Exec("PRAGMA journal_mode=WAL"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store.OnBucketCreation("gonebucket") // the table is never created
|
||||
if err := store.ScanEntries(context.Background(), "/", func(context.Context, *filer.Entry) error { return nil }); err != nil {
|
||||
t.Fatalf("cached-only bucket with a dropped table must not fail the scan: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user