mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-09 15:57:47 +02:00
Chris Lu
a24f4844d3
* mysql: keep S3 list order byte-lexicographic regardless of name column collation ORDER BY name and the name > ? pagination predicate follow the column collation, so a case-insensitive filemeta.name (e.g. utf8mb3_general_ci) returns S3 keys out of byte order and breaks clients that merge two sorted listings. Detect the live name collation at startup; only when it isn't binary, wrap the list comparison, prefix, and ORDER BY in BINARY name so order and pagination stay consistent. Correctly configured utf8mb4_bin tables keep their indexed range scan unchanged, and the operator gets a warning to convert the column. * postgres: keep S3 list order byte-lexicographic regardless of name column collation ORDER BY name and the name > $n pagination predicate follow the column or database collation, so a locale-aware filemeta.name (e.g. the en_US.UTF-8 database default) returns S3 keys out of byte order and breaks clients that merge two sorted listings. Detect the live name collation at startup; only when it isn't byte-ordered, wrap the list comparison, prefix, and ORDER BY in COLLATE "C" so order and pagination stay consistent. A byte-ordered (C/POSIX/C.UTF-8) column keeps its indexed range scan unchanged, and the operator gets a warning to declare the column COLLATE "C".
104 lines
3.2 KiB
Go
104 lines
3.2 KiB
Go
package mysql2
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
_ "github.com/go-sql-driver/mysql"
|
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
|
"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
|
|
"github.com/seaweedfs/seaweedfs/weed/filer/mysql"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
const (
|
|
CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?collation=utf8mb4_bin"
|
|
)
|
|
|
|
var _ filer.BucketAware = (*MysqlStore2)(nil)
|
|
|
|
func init() {
|
|
filer.Stores = append(filer.Stores, &MysqlStore2{})
|
|
}
|
|
|
|
type MysqlStore2 struct {
|
|
abstract_sql.AbstractSqlStore
|
|
}
|
|
|
|
func (store *MysqlStore2) GetName() string {
|
|
return "mysql2"
|
|
}
|
|
|
|
func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
|
|
// Absent key keeps a pooled default; an explicit 0 disables the idle pool.
|
|
configuration.SetDefault(prefix+"connection_max_idle", 2)
|
|
// Default on so minimal configs avoid the duplicate-key roundtrip the
|
|
// inode-index KvPut would otherwise emit on every write.
|
|
configuration.SetDefault(prefix+"enableUpsert", true)
|
|
return store.initialize(
|
|
configuration.GetString(prefix+"createTable"),
|
|
configuration.GetString(prefix+"upsertQuery"),
|
|
configuration.GetBool(prefix+"enableUpsert"),
|
|
configuration.GetString(prefix+"username"),
|
|
configuration.GetString(prefix+"password"),
|
|
configuration.GetString(prefix+"hostname"),
|
|
configuration.GetInt(prefix+"port"),
|
|
configuration.GetString(prefix+"database"),
|
|
configuration.GetInt(prefix+"connection_max_idle"),
|
|
configuration.GetInt(prefix+"connection_max_open"),
|
|
configuration.GetInt(prefix+"connection_max_lifetime_seconds"),
|
|
configuration.GetBool(prefix+"interpolateParams"),
|
|
)
|
|
}
|
|
|
|
func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen,
|
|
maxLifetimeSeconds int, interpolateParams bool) (err error) {
|
|
|
|
store.SupportBucketTable = true
|
|
if !enableUpsert {
|
|
upsertQuery = ""
|
|
} else if upsertQuery == "" {
|
|
upsertQuery = mysql.DefaultUpsertQuery
|
|
}
|
|
gen := &mysql.SqlGenMysql{
|
|
CreateTableSqlTemplate: createTable,
|
|
DropTableSqlTemplate: "DROP TABLE `%s`",
|
|
UpsertQueryTemplate: upsertQuery,
|
|
}
|
|
store.SqlGenerator = gen
|
|
|
|
sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)
|
|
adaptedSqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, "<ADAPTED>", hostname, port, database)
|
|
if interpolateParams {
|
|
sqlUrl += "&interpolateParams=true"
|
|
adaptedSqlUrl += "&interpolateParams=true"
|
|
}
|
|
|
|
var dbErr error
|
|
store.DB, dbErr = sql.Open("mysql", sqlUrl)
|
|
if dbErr != nil {
|
|
store.DB.Close()
|
|
store.DB = nil
|
|
return fmt.Errorf("can not connect to %s error:%v", adaptedSqlUrl, err)
|
|
}
|
|
|
|
store.DB.SetMaxIdleConns(maxIdle)
|
|
store.DB.SetMaxOpenConns(maxOpen)
|
|
store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)
|
|
|
|
if err = store.DB.Ping(); err != nil {
|
|
return fmt.Errorf("connect to %s error:%v", adaptedSqlUrl, err)
|
|
}
|
|
|
|
if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil && !strings.Contains(err.Error(), "table already exist") {
|
|
return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)
|
|
}
|
|
|
|
mysql.ConfigureListOrdering(store.DB, gen)
|
|
|
|
return nil
|
|
}
|