mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-15 11:00:51 +02:00
Chris Lu
a24f4844d3
* mysql: keep S3 list order byte-lexicographic regardless of name column collation ORDER BY name and the name > ? pagination predicate follow the column collation, so a case-insensitive filemeta.name (e.g. utf8mb3_general_ci) returns S3 keys out of byte order and breaks clients that merge two sorted listings. Detect the live name collation at startup; only when it isn't binary, wrap the list comparison, prefix, and ORDER BY in BINARY name so order and pagination stay consistent. Correctly configured utf8mb4_bin tables keep their indexed range scan unchanged, and the operator gets a warning to convert the column. * postgres: keep S3 list order byte-lexicographic regardless of name column collation ORDER BY name and the name > $n pagination predicate follow the column or database collation, so a locale-aware filemeta.name (e.g. the en_US.UTF-8 database default) returns S3 keys out of byte order and breaks clients that merge two sorted listings. Detect the live name collation at startup; only when it isn't byte-ordered, wrap the list comparison, prefix, and ORDER BY in COLLATE "C" so order and pagination stay consistent. A byte-ordered (C/POSIX/C.UTF-8) column keeps its indexed range scan unchanged, and the operator gets a warning to declare the column COLLATE "C".
42 lines
1.7 KiB
Go
42 lines
1.7 KiB
Go
package mysql
|
|
|
|
import (
|
|
"database/sql"
|
|
"strings"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
)
|
|
|
|
// ConfigureListOrdering switches the list queries to BINARY byte ordering when
|
|
// the live filemeta.name collation is not binary, so S3 ListObjectsV2 stays
|
|
// lexicographic. The fallback costs a filesort, so warn the operator to fix it.
|
|
func ConfigureListOrdering(db *sql.DB, gen *SqlGenMysql) {
|
|
collation, isBinary, err := nameColumnCollation(db)
|
|
if err != nil {
|
|
glog.V(1).Infof("mysql: skip filemeta.name collation check: %v", err)
|
|
return
|
|
}
|
|
if isBinary {
|
|
return
|
|
}
|
|
gen.ForceBinaryCollation = true
|
|
glog.Warningf("mysql: filemeta.name collation %q is not binary, so S3 list order is not byte-lexicographic and clients that merge sorted listings may report spurious diffs. Falling back to a slower BINARY sort; convert the name column to a *_bin collation (e.g. utf8mb4_bin) for correct, indexed ordering.", collation)
|
|
}
|
|
|
|
func nameColumnCollation(db *sql.DB) (collation string, isBinary bool, err error) {
|
|
row := db.QueryRow("SELECT COLLATION_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = 'name'", abstract_sql.DEFAULT_TABLE)
|
|
var c sql.NullString
|
|
if err = row.Scan(&c); err != nil {
|
|
return "", false, err
|
|
}
|
|
return c.String, isBinaryCollation(c.String), nil
|
|
}
|
|
|
|
// isBinaryCollation reports whether a collation orders by byte value: a NULL
|
|
// collation (BINARY column) or any *_bin collation.
|
|
func isBinaryCollation(collation string) bool {
|
|
c := strings.ToLower(strings.TrimSpace(collation))
|
|
return c == "" || c == "binary" || strings.HasSuffix(c, "_bin")
|
|
}
|