mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-23 14:30:46 +02:00
Jaehoon KimandChris Lu
744ab586c5
* shell: volume.fsck reads the filer from one store snapshot when it can volume.fsck collects the filer's chunk references by walking directory by directory, so each directory is seen at a different time. A file renamed from a directory not yet listed into one already listed is never seen, its chunks look like orphans, and -reallyDeleteFromVolume deletes live data. Checkpoint rotation and write-to-tmp-then-rename hit this. Add an optional filer store capability, EntryScanner, that visits every entry below a directory from one consistent snapshot. The SQL stores implement it with one REPEATABLE READ read-only transaction read in primary-key pages, so a concurrent rename is seen under exactly one of its names. Key-value rows are skipped by their dirhash, a bucket directory whose table is missing fails the scan (from the snapshot it cannot be told from one dropped a moment ago), and hard links are resolved in the same transaction. SQLite offers it only in WAL mode, on its own connection. TraverseBfsMetadata serves the snapshot when asked via request metadata sw-traverse-mode=snapshot and reports the mode in a response header; without the metadata it walks as before, so peer bootstrap and admin export are unchanged. Snapshot responses carry no entry_ts_ns, whose per-entry lookup would cost more than the scan. volume.fsck requests the snapshot and falls back to the walk on an older filer or a store that cannot scan. * filer, shell: fail a scan that cannot start, not fall back to the walk A transient BeginTx failure and a preliminary GetEntry error were treated like a missing capability or missing root, so fsck would take the directory walk - the very path that misses renamed-away entries. Only a genuinely unsupported scan or a confirmed-absent root may fall back. * filer: only snapshot-listed bucket names keep a missing table fatal A cached-only name, whose bucket may have been deleted between copying the cache and opening the snapshot, is skipped when its table is gone. * filer: roll back to a savepoint before skipping a cached-only table On PostgreSQL a failed statement aborts the snapshot transaction, so ignoring a missing-table error without the savepoint still left every later cached-only table unreadable. --------- Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
106 lines
4.1 KiB
Go
106 lines
4.1 KiB
Go
package postgres
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
_ "github.com/jackc/pgx/v5/stdlib"
|
|
"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
|
|
)
|
|
|
|
type SqlGenPostgres struct {
|
|
CreateTableSqlTemplate string
|
|
DropTableSqlTemplate string
|
|
UpsertQueryTemplate string
|
|
// Force byte ordering on a locale-aware name column; see ConfigureListOrdering.
|
|
ForceBinaryCollation bool
|
|
}
|
|
|
|
// DefaultUpsertQuery keeps INSERTs idempotent so a duplicate-key failure
|
|
// (23505) cannot poison the surrounding transaction (25P02). Used when the
|
|
// user enables upsert but does not provide their own template.
|
|
const DefaultUpsertQuery = `INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash, name) DO UPDATE SET directory=EXCLUDED.directory, meta=EXCLUDED.meta`
|
|
|
|
// DefaultCreateTableQuery is used when createTable is left unset; an empty
|
|
// template would otherwise render as %!(EXTRA ...) garbage SQL.
|
|
const DefaultCreateTableQuery = `CREATE TABLE IF NOT EXISTS "%s" (dirhash BIGINT, name VARCHAR(65535), directory VARCHAR(65535), meta bytea, PRIMARY KEY (dirhash, name))`
|
|
|
|
var (
|
|
_ = abstract_sql.SqlGenerator(&SqlGenPostgres{})
|
|
)
|
|
|
|
// ResolveCreateTableQuery normalizes the createTable config value. A boolean
|
|
// true (read by viper as the string "true") selects the default template.
|
|
// An empty or false value returns an empty string so the caller can skip
|
|
// table creation. Any other value is treated as a custom SQL template.
|
|
func ResolveCreateTableQuery(createTable string) string {
|
|
switch createTable {
|
|
case "true":
|
|
return DefaultCreateTableQuery
|
|
case "false", "":
|
|
return ""
|
|
default:
|
|
return createTable
|
|
}
|
|
}
|
|
|
|
func (gen *SqlGenPostgres) GetSqlInsert(tableName string) string {
|
|
if gen.UpsertQueryTemplate != "" {
|
|
return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)
|
|
} else {
|
|
return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, tableName)
|
|
}
|
|
}
|
|
|
|
func (gen *SqlGenPostgres) GetSqlUpdate(tableName string) string {
|
|
return fmt.Sprintf(`UPDATE "%s" SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4`, tableName)
|
|
}
|
|
|
|
func (gen *SqlGenPostgres) GetSqlFind(tableName string) string {
|
|
return fmt.Sprintf(`SELECT meta FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, tableName)
|
|
}
|
|
|
|
func (gen *SqlGenPostgres) GetSqlDelete(tableName string) string {
|
|
return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, tableName)
|
|
}
|
|
|
|
func (gen *SqlGenPostgres) GetSqlDeleteFolderChildren(tableName string) string {
|
|
return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND directory=$2`, tableName)
|
|
}
|
|
|
|
// nameExpr forces byte ordering on a locale-aware column via COLLATE "C".
|
|
func (gen *SqlGenPostgres) nameExpr() string {
|
|
if gen.ForceBinaryCollation {
|
|
return `name COLLATE "C"`
|
|
}
|
|
return "name"
|
|
}
|
|
|
|
// GetSqlScanPage uses a row-value comparison, one primary-key range seek on
|
|
// PostgreSQL. The name keeps the column's own collation (COLLATE "C" would
|
|
// sort every page); the scan promises no lexical order.
|
|
func (gen *SqlGenPostgres) GetSqlScanPage(tableName string, dirhash int64, name string, limit int) (string, []interface{}) {
|
|
return fmt.Sprintf(`SELECT dirhash, name, directory, meta FROM "%s" WHERE (dirhash, name) > ($1, $2) ORDER BY dirhash ASC, name ASC LIMIT $3`, tableName),
|
|
[]interface{}{dirhash, name, limit}
|
|
}
|
|
|
|
func (gen *SqlGenPostgres) GetSqlListExclusive(tableName string) string {
|
|
name := gen.nameExpr()
|
|
return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>$2 AND directory=$3 AND %s like $4 ESCAPE '!' ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)
|
|
}
|
|
|
|
func (gen *SqlGenPostgres) GetSqlListInclusive(tableName string) string {
|
|
name := gen.nameExpr()
|
|
return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>=$2 AND directory=$3 AND %s like $4 ESCAPE '!' ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)
|
|
}
|
|
|
|
func (gen *SqlGenPostgres) GetSqlCreateTable(tableName string) string {
|
|
if gen.CreateTableSqlTemplate == "" {
|
|
return ""
|
|
}
|
|
return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)
|
|
}
|
|
|
|
func (gen *SqlGenPostgres) GetSqlDropTable(tableName string) string {
|
|
return fmt.Sprintf(gen.DropTableSqlTemplate, tableName)
|
|
}
|