From a5ccefb661c6a3dd19fe4cb4ba3cdf419eff47db Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 16 Oct 2026 08:51:15 +0900 Subject: [PATCH] filer: escape LIKE wildcards in SQL list prefixes (#11757) * filer: escape LIKE wildcards in SQL list prefixes ListDirectoryPrefixedEntries bound prefix+"%" straight into a LIKE clause, so '_' and '%' in object names (e.g. Langfuse keys like call_03_...) acted as wildcards: wrong results and a ~40x slower range scan because (dirhash, name) can no longer be narrowed. Escape '!'/'%'/'_' in the prefix and declare ESCAPE '!' in the generated list SQL so every abstract_sql backend (mysql, mysql2, postgres, postgres2, sqlite) matches literally with identical semantics. '!' is used instead of '\' because SQLite shares SqlGenMysql but would reject the MySQL-style '\\' literal, and '\' would still need escaping itself. * filer: escape LIKE wildcards in YDB list prefixes Same wildcard-in-prefix bug as the abstract_sql backends: YQL LIKE treated '_' and '%' in the prefix as wildcards. Apply the shared LikePattern helper and declare ESCAPE '!' (YQL allows any escape char except '%', '_' and '\'). * test: cover literal '_'/'%'/'!' prefixes in sqlite filer listing End-to-end regression test through SqliteStore against a real database; runs in CI under the sqlite build tag. --- weed/filer/abstract_sql/abstract_sql_store.go | 10 ++- .../abstract_sql/abstract_sql_store_test.go | 18 +++++ weed/filer/mysql/mysql_sql_gen.go | 4 +- weed/filer/mysql/mysql_sql_gen_test.go | 9 +++ weed/filer/postgres/postgres_sql_gen.go | 4 +- weed/filer/postgres/postgres_sql_gen_test.go | 9 +++ weed/filer/sqlite/sqlite_like_escape_test.go | 75 +++++++++++++++++++ weed/filer/ydb/ydb_queries.go | 4 +- weed/filer/ydb/ydb_store.go | 2 +- 9 files changed, 127 insertions(+), 8 deletions(-) create mode 100644 weed/filer/sqlite/sqlite_like_escape_test.go diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index 74a6b162f..e35872774 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -390,6 +390,14 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat return doDeleteFolderChildren() } +var likeEscaper = strings.NewReplacer("!", "!!", "%", "!%", "_", "!_") + +// LikePattern escapes LIKE wildcards so the prefix matches literally; +// '!' pairs with the ESCAPE '!' clause emitted by the SqlGenerators. +func LikePattern(prefix string) string { + return likeEscaper.Replace(prefix) + "%" +} + func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { db, bucket, shortPath, err := store.getTxOrDB(ctx, dirPath, true) @@ -402,7 +410,7 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, sqlText = store.GetSqlListInclusive(bucket) } - rows, err := db.QueryContext(ctx, sqlText, util.HashStringToLong(string(shortPath)), startFileName, string(shortPath), prefix+"%", limit+1) + rows, err := db.QueryContext(ctx, sqlText, util.HashStringToLong(string(shortPath)), startFileName, string(shortPath), LikePattern(prefix), limit+1) if err != nil { return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) } diff --git a/weed/filer/abstract_sql/abstract_sql_store_test.go b/weed/filer/abstract_sql/abstract_sql_store_test.go index 540183e9e..818360327 100644 --- a/weed/filer/abstract_sql/abstract_sql_store_test.go +++ b/weed/filer/abstract_sql/abstract_sql_store_test.go @@ -52,6 +52,24 @@ func TestGetTxOrDBRealBucket(t *testing.T) { } } +func TestLikePattern(t *testing.T) { + cases := []struct { + prefix, want string + }{ + {"", "%"}, + {"plain", "plain%"}, + {"call_03_KNl", "call!_03!_KNl%"}, + {"100%", "100!%%"}, + {"a!b", "a!!b%"}, + {`a\b`, `a\b%`}, + } + for _, c := range cases { + if got := LikePattern(c.prefix); got != c.want { + t.Errorf("LikePattern(%q) = %q, want %q", c.prefix, got, c.want) + } + } +} + func TestSplitPoolForKv(t *testing.T) { cases := []struct { maxOpen, mainOpen, kvOpen int diff --git a/weed/filer/mysql/mysql_sql_gen.go b/weed/filer/mysql/mysql_sql_gen.go index 52c9e710c..049ea685a 100644 --- a/weed/filer/mysql/mysql_sql_gen.go +++ b/weed/filer/mysql/mysql_sql_gen.go @@ -64,12 +64,12 @@ func (gen *SqlGenMysql) nameExpr() string { func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string { name := gen.nameExpr() - return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s > ? AND `directory` = ? AND %s LIKE ? ORDER BY %s ASC LIMIT ?", tableName, name, name, name) + return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s > ? AND `directory` = ? AND %s LIKE ? ESCAPE '!' ORDER BY %s ASC LIMIT ?", tableName, name, name, name) } func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string { name := gen.nameExpr() - return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s >= ? AND `directory` = ? AND %s LIKE ? ORDER BY %s ASC LIMIT ?", tableName, name, name, name) + return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s >= ? AND `directory` = ? AND %s LIKE ? ESCAPE '!' ORDER BY %s ASC LIMIT ?", tableName, name, name, name) } func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string { diff --git a/weed/filer/mysql/mysql_sql_gen_test.go b/weed/filer/mysql/mysql_sql_gen_test.go index 3f01965f3..d032fc7b7 100644 --- a/weed/filer/mysql/mysql_sql_gen_test.go +++ b/weed/filer/mysql/mysql_sql_gen_test.go @@ -65,6 +65,15 @@ func TestListSqlBinaryOrderingOnNonBinaryColumn(t *testing.T) { } } +func TestListSqlEscapesLikeWildcards(t *testing.T) { + gen := &SqlGenMysql{} + for _, got := range []string{gen.GetSqlListExclusive("filemeta"), gen.GetSqlListInclusive("filemeta")} { + if !strings.Contains(got, "LIKE ? ESCAPE '!'") { + t.Fatalf("list query must declare the escape char matching LikePattern, got: %s", got) + } + } +} + func TestIsBinaryCollation(t *testing.T) { binary := []string{"", "binary", "utf8mb4_bin", "utf8mb3_bin", "latin1_bin", "UTF8MB4_BIN"} for _, c := range binary { diff --git a/weed/filer/postgres/postgres_sql_gen.go b/weed/filer/postgres/postgres_sql_gen.go index 0f0064b3e..9ff50d96a 100644 --- a/weed/filer/postgres/postgres_sql_gen.go +++ b/weed/filer/postgres/postgres_sql_gen.go @@ -77,12 +77,12 @@ func (gen *SqlGenPostgres) nameExpr() string { func (gen *SqlGenPostgres) GetSqlListExclusive(tableName string) string { name := gen.nameExpr() - return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>$2 AND directory=$3 AND %s like $4 ORDER BY %s ASC LIMIT $5`, tableName, name, name, name) + return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>$2 AND directory=$3 AND %s like $4 ESCAPE '!' ORDER BY %s ASC LIMIT $5`, tableName, name, name, name) } func (gen *SqlGenPostgres) GetSqlListInclusive(tableName string) string { name := gen.nameExpr() - return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>=$2 AND directory=$3 AND %s like $4 ORDER BY %s ASC LIMIT $5`, tableName, name, name, name) + return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>=$2 AND directory=$3 AND %s like $4 ESCAPE '!' ORDER BY %s ASC LIMIT $5`, tableName, name, name, name) } func (gen *SqlGenPostgres) GetSqlCreateTable(tableName string) string { diff --git a/weed/filer/postgres/postgres_sql_gen_test.go b/weed/filer/postgres/postgres_sql_gen_test.go index 1436d36a9..1cb947bd7 100644 --- a/weed/filer/postgres/postgres_sql_gen_test.go +++ b/weed/filer/postgres/postgres_sql_gen_test.go @@ -62,6 +62,15 @@ func TestListSqlBinaryOrderingOnNonBinaryColumn(t *testing.T) { } } +func TestListSqlEscapesLikeWildcards(t *testing.T) { + gen := &SqlGenPostgres{} + for _, got := range []string{gen.GetSqlListExclusive("filemeta"), gen.GetSqlListInclusive("filemeta")} { + if !strings.Contains(got, "like $4 ESCAPE '!'") { + t.Fatalf("list query must declare the escape char matching LikePattern, got: %s", got) + } + } +} + func TestIsByteOrderedCollation(t *testing.T) { byteOrdered := []string{"C", "POSIX", "c", "C.UTF-8", "C.utf8"} for _, c := range byteOrdered { diff --git a/weed/filer/sqlite/sqlite_like_escape_test.go b/weed/filer/sqlite/sqlite_like_escape_test.go new file mode 100644 index 000000000..d2d39bf3c --- /dev/null +++ b/weed/filer/sqlite/sqlite_like_escape_test.go @@ -0,0 +1,75 @@ +//go:build sqlite + +package sqlite + +import ( + "context" + "database/sql" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/filer/mysql" + "github.com/seaweedfs/seaweedfs/weed/util" + _ "modernc.org/sqlite" +) + +func TestListDirectoryPrefixedEntriesEscapesLikeWildcards(t *testing.T) { + db, err := sql.Open("sqlite", "file::memory:?cache=shared") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + _, err = db.Exec(`CREATE TABLE filemeta (dirhash BIGINT, name VARCHAR(1000), directory TEXT, meta BLOB, PRIMARY KEY (dirhash, name))`) + if err != nil { + t.Fatal(err) + } + + store := &SqliteStore{} + store.SqlGenerator = &mysql.SqlGenMysql{} + store.DB = db + + ctx := context.Background() + names := []string{ + "call_03_KNlHPt6grkZMe4WZVwQi9710", // target: literal _ prefix + "callX03XKNlZZZ", // would match if _ were a wildcard + "call_99_other", + "100%_done", // % and _ in the name + "100aXdone", // would match prefix "100%_" if unescaped + "bang!file", + } + for _, n := range names { + entry := &filer.Entry{FullPath: util.FullPath("/dir/" + n)} + if err := store.InsertEntry(ctx, entry); err != nil { + t.Fatalf("insert %s: %v", n, err) + } + } + + list := func(prefix string) []string { + var got []string + _, err := store.ListDirectoryPrefixedEntries(ctx, "/dir", "", false, 100, prefix, func(e *filer.Entry) (bool, error) { + got = append(got, e.Name()) + return true, nil + }) + if err != nil { + t.Fatalf("list %q: %v", prefix, err) + } + return got + } + + if got := list("call_03_"); len(got) != 1 || got[0] != "call_03_KNlHPt6grkZMe4WZVwQi9710" { + t.Fatalf("prefix with _: got %v", got) + } + if got := list("call_"); len(got) != 2 { + t.Fatalf("prefix call_: got %v", got) + } + if got := list("100%_"); len(got) != 1 || got[0] != "100%_done" { + t.Fatalf("prefix with %%: got %v", got) + } + if got := list("bang!"); len(got) != 1 || got[0] != "bang!file" { + t.Fatalf("prefix with !: got %v", got) + } + if got := list(""); len(got) != len(names) { + t.Fatalf("empty prefix: got %v", got) + } +} diff --git a/weed/filer/ydb/ydb_queries.go b/weed/filer/ydb/ydb_queries.go index 1118cceb6..2b57a069c 100644 --- a/weed/filer/ydb/ydb_queries.go +++ b/weed/filer/ydb/ydb_queries.go @@ -55,7 +55,7 @@ const ( SELECT name, meta FROM ` + asql.DEFAULT_TABLE + ` - WHERE dir_hash = $dir_hash AND directory = $directory and name > $start_name and name LIKE $prefix + WHERE dir_hash = $dir_hash AND directory = $directory and name > $start_name and name LIKE $prefix ESCAPE '!' ORDER BY name ASC LIMIT $limit;` listInclusiveDirectoryQuery = ` @@ -68,6 +68,6 @@ const ( SELECT name, meta FROM ` + asql.DEFAULT_TABLE + ` - WHERE dir_hash = $dir_hash AND directory = $directory and name >= $start_name and name LIKE $prefix + WHERE dir_hash = $dir_hash AND directory = $directory and name >= $start_name and name LIKE $prefix ESCAPE '!' ORDER BY name ASC LIMIT $limit;` ) diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go index e5d14b5ff..459c6d2b7 100644 --- a/weed/filer/ydb/ydb_store.go +++ b/weed/filer/ydb/ydb_store.go @@ -288,7 +288,7 @@ func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))), table.ValueParam("$directory", types.UTF8Value(*shortDir)), table.ValueParam("$start_name", types.UTF8Value(startFileName)), - table.ValueParam("$prefix", types.UTF8Value(prefix+"%")), + table.ValueParam("$prefix", types.UTF8Value(abstract_sql.LikePattern(prefix))), table.ValueParam("$limit", types.Uint64Value(uint64(chunkLimit))), )