From cb9fcd39d2aeaba46ec6384e87b112020e9cdb70 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 8 Sep 2026 16:17:02 -0700 Subject: [PATCH] filer/postgres: create filemeta table on startup via createTable config (#11229) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * filer/postgres: create default filemeta table on startup The postgres filer store hardcoded CreateTableSqlTemplate to empty and never created the filemeta table, unlike postgres2/mysql2/sqlite which all create it during Initialize. Users had to create the table manually or the filer would crash loop with "relation filemeta does not exist". Read the createTable config option (same as postgres2), default to DefaultCreateTableQuery when unset, and execute CREATE TABLE IF NOT EXISTS on the default table after the connection pool is established. SupportBucketTable stays false so per-bucket table creation remains a no-op; only the shared filemeta table is created, via a direct ExecContext since AbstractSqlStore.CreateTable short-circuits without bucket support. * filer/postgres: accept boolean createTable = true/false viper reads a TOML boolean as the string "true"/"false" via GetString, so createTable = true was being used as a SQL template and failed. Add ResolveCreateTableQuery to normalize the value: true and empty select the default template, false disables table creation, anything else is a custom template. Both postgres and postgres2 now use it, and both skip the CREATE TABLE call when the resolved template is empty. * scaffold: document createTable option for postgres filer store Replace the commented-out CREATE TABLE SQL in the [postgres] scaffold with a createTable config hint, matching the [postgres2] section. Users no longer need to manually create the filemeta table before starting the filer. * filer/postgres: make createTable opt-in for postgres, keep postgres2 default The previous commit defaulted postgres to create the filemeta table even when createTable was unset, which could break existing deployments whose DB user lacks CREATE TABLE privileges. ResolveCreateTableQuery now returns empty for an unset value so postgres only creates the table when createTable is explicitly true or a custom template — preserving the prior no-DDL behaviour for existing configurations. postgres2 keeps its existing always-create default: it defaults an empty resolved value to DefaultCreateTableQuery, and only skips when createTable is explicitly false. * filer/postgres2: simplify createTable handling, document all modes Drop the false opt-out from postgres2 — it only skipped the default table while per-bucket CreateTable still ran, leaving restricted DB roles broken on bucket access. postgres2 now accepts true the same way (defaulting to DefaultCreateTableQuery) and keeps its existing always-create behaviour for every other value, matching the original semantics. The scaffold comment now documents true/false/custom for the postgres section so users know false (or unset) is the backward-compatible default. * filer/postgres2: normalize false via ResolveCreateTableQuery postgres2 only handled "" and "true", leaving createTable = false as the literal string "false" which CreateTable then executed as invalid SQL. Route it through ResolveCreateTableQuery (which maps false to empty) and default the empty result to DefaultCreateTableQuery, so false is treated the same as unset for the bucket-aware store. * filer/postgres2: honor createTable = false for default table postgres2 treated false the same as unset and always created the default filemeta table, failing startup for restricted DB roles that explicitly opted out. Track the original false value before ResolveCreateTableQuery collapses it to empty, and skip the default CreateTable call when set. Per-bucket table creation is unaffected — it is a runtime requirement of the bucket-aware store. Users who need to suppress all DDL should use the postgres (non-bucket) store with createTable unset. * filer/postgres2: disable bucket tables when createTable = false Setting SupportBucketTable = false when createTable is explicitly false makes AbstractSqlStore.CreateTable a no-op (it already returns nil when SupportBucketTable is false), so neither the default filemeta table nor per-bucket tables are created. The template stays empty and no DDL runs, honouring the opt-out for restricted DB roles. All data routes to the pre-provisioned filemeta table, matching the postgres (non-bucket) store. * filer: suppress DDL without disabling bucket routing Setting SupportBucketTable = false when createTable = false also disabled per-bucket routing, hiding objects in pre-provisioned per-bucket tables. Keep SupportBucketTable true and instead skip the CREATE TABLE execution when the resolved template is empty. GetSqlCreateTable now returns empty for both postgres and mysql SQL generators when CreateTableSqlTemplate is empty, and AbstractSqlStore.CreateTable skips the ExecContext call when the SQL is empty. This preserves bucket routing while suppressing all DDL for users who explicitly set createTable = false and pre-provision their tables. * filer: add SkipDDL to suppress CREATE and DROP without disabling routing createTable = false with SupportBucketTable = true preserved bucket routing but deleteTable still executed DROP TABLE on bucket deletion, dropping externally managed tables. CanDropWholeBucket also returned true, so the S3 layer tried whole-table drops instead of row-by-row deletes. Add a SkipDDL flag to AbstractSqlStore, independent of SupportBucketTable. CreateTable and deleteTable both skip when SkipDDL is set, and CanDropWholeBucket returns false so bucket deletion falls back to row-by-row metadata deletes. postgres2 sets SkipDDL when createTable is explicitly false — bucket routing is preserved, no DDL runs. * filer: fall back to row-by-row delete when CanDropWholeBucket is false DeleteFolderChildren took the whole-table drop path whenever the path was a bucket root, even when SkipDDL made deleteTable a no-op. The no-op returned nil, the caller returned early, and rows inserted after the recursive enumeration survived the bucket deletion. Gate the whole-table drop on CanDropWholeBucket so the row-by-row DeleteFolderChildren SQL runs when SkipDDL is set, removing all metadata without issuing DROP TABLE. --- weed/command/scaffold/filer.toml | 10 +++------- weed/filer/abstract_sql/abstract_sql_store.go | 15 ++++++++++----- weed/filer/mysql/mysql_sql_gen.go | 3 +++ weed/filer/postgres/postgres_sql_gen.go | 18 ++++++++++++++++++ weed/filer/postgres/postgres_sql_gen_test.go | 16 ++++++++++++++++ weed/filer/postgres/postgres_store.go | 14 ++++++++++++-- weed/filer/postgres2/postgres2_store.go | 10 +++++++--- 7 files changed, 69 insertions(+), 17 deletions(-) diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 397b93887..57b4d85db 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -106,13 +106,9 @@ enableUpsert = true upsertQuery = """INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES (?,?,?,?) AS `new` ON DUPLICATE KEY UPDATE `meta` = `new`.`meta`""" [postgres] # or cockroachdb, YugabyteDB -# CREATE TABLE IF NOT EXISTS filemeta ( -# dirhash BIGINT, -# name VARCHAR(65535), -# directory VARCHAR(65535), -# meta bytea, -# PRIMARY KEY (dirhash, name) -# ); +# createTable = true # auto-create filemeta with the default schema +# createTable = false # skip table creation (default; for restricted DB roles) +# createTable = """CREATE TABLE IF NOT EXISTS "%s" (...)""" # custom template enabled = false hostname = "localhost" port = 5432 diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index b35a24e8e..74a6b162f 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -32,6 +32,7 @@ type AbstractSqlStore struct { DB *sql.DB KvDB *sql.DB SupportBucketTable bool + SkipDDL bool dbs map[string]bool dbsLock sync.Mutex RetryableErrorCallback func(err error) bool @@ -40,7 +41,7 @@ type AbstractSqlStore struct { var _ filer.BucketAware = (*AbstractSqlStore)(nil) func (store *AbstractSqlStore) CanDropWholeBucket() bool { - return store.SupportBucketTable + return store.SupportBucketTable && !store.SkipDDL } func (store *AbstractSqlStore) OnBucketCreation(bucket string) { store.dbsLock.Lock() @@ -356,7 +357,7 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat return fmt.Errorf("findDB %s : %w", fullpath, err) } - if isValidBucket(bucket) && shortPath == "/" { + if isValidBucket(bucket) && shortPath == "/" && store.CanDropWholeBucket() { if err = store.deleteTable(ctx, bucket); err == nil { store.dbsLock.Lock() delete(store.dbs, bucket) @@ -458,15 +459,19 @@ func isValidBucket(bucket string) bool { } func (store *AbstractSqlStore) CreateTable(ctx context.Context, bucket string) error { - if !store.SupportBucketTable { + if !store.SupportBucketTable || store.SkipDDL { return nil } - _, err := store.DB.ExecContext(ctx, store.SqlGenerator.GetSqlCreateTable(bucket)) + sql := store.SqlGenerator.GetSqlCreateTable(bucket) + if sql == "" { + return nil + } + _, err := store.DB.ExecContext(ctx, sql) return err } func (store *AbstractSqlStore) deleteTable(ctx context.Context, bucket string) error { - if !store.SupportBucketTable { + if !store.SupportBucketTable || store.SkipDDL { return nil } _, err := store.DB.ExecContext(ctx, store.SqlGenerator.GetSqlDropTable(bucket)) diff --git a/weed/filer/mysql/mysql_sql_gen.go b/weed/filer/mysql/mysql_sql_gen.go index 610235f7c..52c9e710c 100644 --- a/weed/filer/mysql/mysql_sql_gen.go +++ b/weed/filer/mysql/mysql_sql_gen.go @@ -73,6 +73,9 @@ func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string { } func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string { + if gen.CreateTableSqlTemplate == "" { + return "" + } return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName) } diff --git a/weed/filer/postgres/postgres_sql_gen.go b/weed/filer/postgres/postgres_sql_gen.go index e308e2691..0f0064b3e 100644 --- a/weed/filer/postgres/postgres_sql_gen.go +++ b/weed/filer/postgres/postgres_sql_gen.go @@ -28,6 +28,21 @@ var ( _ = abstract_sql.SqlGenerator(&SqlGenPostgres{}) ) +// ResolveCreateTableQuery normalizes the createTable config value. A boolean +// true (read by viper as the string "true") selects the default template. +// An empty or false value returns an empty string so the caller can skip +// table creation. Any other value is treated as a custom SQL template. +func ResolveCreateTableQuery(createTable string) string { + switch createTable { + case "true": + return DefaultCreateTableQuery + case "false", "": + return "" + default: + return createTable + } +} + func (gen *SqlGenPostgres) GetSqlInsert(tableName string) string { if gen.UpsertQueryTemplate != "" { return fmt.Sprintf(gen.UpsertQueryTemplate, tableName) @@ -71,6 +86,9 @@ func (gen *SqlGenPostgres) GetSqlListInclusive(tableName string) string { } func (gen *SqlGenPostgres) GetSqlCreateTable(tableName string) string { + if gen.CreateTableSqlTemplate == "" { + return "" + } return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName) } diff --git a/weed/filer/postgres/postgres_sql_gen_test.go b/weed/filer/postgres/postgres_sql_gen_test.go index 72b11e657..1436d36a9 100644 --- a/weed/filer/postgres/postgres_sql_gen_test.go +++ b/weed/filer/postgres/postgres_sql_gen_test.go @@ -76,3 +76,19 @@ func TestIsByteOrderedCollation(t *testing.T) { } } } + +func TestResolveCreateTableQuery(t *testing.T) { + cases := []struct { + in, want string + }{ + {"", ""}, + {"true", DefaultCreateTableQuery}, + {"false", ""}, + {"CREATE TABLE custom", "CREATE TABLE custom"}, + } + for _, c := range cases { + if got := ResolveCreateTableQuery(c.in); got != c.want { + t.Fatalf("ResolveCreateTableQuery(%q) = %q, want %q", c.in, got, c.want) + } + } +} diff --git a/weed/filer/postgres/postgres_store.go b/weed/filer/postgres/postgres_store.go index 73926071d..fe1e57478 100644 --- a/weed/filer/postgres/postgres_store.go +++ b/weed/filer/postgres/postgres_store.go @@ -8,7 +8,9 @@ package postgres import ( + "context" "database/sql" + "fmt" "strconv" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -39,6 +41,7 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix // poisoning on Postgres; an explicit false still disables it. configuration.SetDefault(prefix+"enableUpsert", true) return store.initialize( + configuration.GetString(prefix+"createTable"), configuration.GetString(prefix+"upsertQuery"), configuration.GetBool(prefix+"enableUpsert"), configuration.GetString(prefix+"username"), @@ -59,16 +62,17 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix ) } -func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) { +func (store *PostgresStore) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) { store.SupportBucketTable = false + createTable = ResolveCreateTableQuery(createTable) if !enableUpsert { upsertQuery = "" } else if upsertQuery == "" { upsertQuery = DefaultUpsertQuery } gen := &SqlGenPostgres{ - CreateTableSqlTemplate: "", + CreateTableSqlTemplate: createTable, DropTableSqlTemplate: `drop table if exists "%s"`, UpsertQueryTemplate: upsertQuery, } @@ -126,6 +130,12 @@ func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, us return err } + if createTable != "" { + if _, err = store.DB.ExecContext(context.Background(), gen.GetSqlCreateTable(abstract_sql.DEFAULT_TABLE)); err != nil { + return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err) + } + } + ConfigureListOrdering(store.DB, gen) return nil diff --git a/weed/filer/postgres2/postgres2_store.go b/weed/filer/postgres2/postgres2_store.go index dfd7ce387..d523a2abb 100644 --- a/weed/filer/postgres2/postgres2_store.go +++ b/weed/filer/postgres2/postgres2_store.go @@ -68,7 +68,9 @@ func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) { store.SupportBucketTable = true - if createTable == "" { + store.SkipDDL = createTable == "false" + createTable = postgres.ResolveCreateTableQuery(createTable) + if createTable == "" && !store.SkipDDL { createTable = postgres.DefaultCreateTableQuery } if !enableUpsert { @@ -135,8 +137,10 @@ func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableU return err } - if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil { - return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err) + if !store.SkipDDL { + if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil { + return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err) + } } postgres.ConfigureListOrdering(store.DB, gen)