mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-08 15:41:15 +02:00
Chris Lu
9f6efc8b53
* filer: give the SQL stores' key-value reads their own connections A listing holds the connection its rows are on for the whole iteration, and FilerStoreWrapper calls maybeReadHardLink -> KvGet from inside that iteration, so a hard-linked entry needs a second connection while the first is still busy. Out of one bounded pool that is a deadlock: the listings fill the pool and then wait for a connection none of them will release, and the wrapper's context.WithoutCancel leaves the waiters without a deadline, so the filer stays wedged rather than erroring. The sqlite store shows it at its sharpest -- it allows a single connection, so one listing over one hard-linked entry never returns. On postgres with connection_max_open = 50, 60 concurrent listings over hard-linked entries made no progress at all. Key-value reads now run on their own pool, carved out of connection_max_open rather than added to it, so the operator's cap still bounds what the store opens against the database. An unbounded pool keeps a single pool: nothing can wait there. sqlite's single connection becomes two, one per pool, and its writes get a busy timeout so a write that meets the reader waits instead of failing. Claude-Session: https://claude.ai/code/session_018DWwctzD4T2DmnczPRM47t * sqlite: keep both pools on one database, whatever the dbFile spells A dbFile that already carries URI options got a second "?" appended, which the driver reads as part of the preceding option value, and a bare :memory: is private to each connection, so the key-value pool would open its own empty database and every key-value operation would fail on a missing filemeta. Claude-Session: https://claude.ai/code/session_018DWwctzD4T2DmnczPRM47t * sqlite: assert the busy timeout on the in-memory DSN too Claude-Session: https://claude.ai/code/session_018DWwctzD4T2DmnczPRM47t
104 lines
2.6 KiB
Go
104 lines
2.6 KiB
Go
//go:build (linux || darwin || windows) && sqlite
|
|
|
|
// limited GOOS due to modernc.org/libc/unistd
|
|
|
|
package sqlite
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
|
"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
|
|
"github.com/seaweedfs/seaweedfs/weed/filer/mysql"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
func init() {
|
|
filer.Stores = append(filer.Stores, &SqliteStore{})
|
|
}
|
|
|
|
type SqliteStore struct {
|
|
abstract_sql.AbstractSqlStore
|
|
}
|
|
|
|
func (store *SqliteStore) GetName() string {
|
|
return "sqlite"
|
|
}
|
|
|
|
func (store *SqliteStore) Initialize(configuration util.Configuration, prefix string) (err error) {
|
|
dbFile := configuration.GetString(prefix + "dbFile")
|
|
createTable := `CREATE TABLE IF NOT EXISTS "%s" (
|
|
dirhash BIGINT,
|
|
name VARCHAR(1000),
|
|
directory TEXT,
|
|
meta BLOB,
|
|
PRIMARY KEY (dirhash, name)
|
|
) WITHOUT ROWID;`
|
|
upsertQuery := `INSERT INTO "%s"(dirhash,name,directory,meta)VALUES(?,?,?,?)
|
|
ON CONFLICT(dirhash,name) DO UPDATE SET
|
|
directory=excluded.directory,
|
|
meta=excluded.meta;
|
|
`
|
|
return store.initialize(
|
|
dbFile,
|
|
createTable,
|
|
upsertQuery,
|
|
)
|
|
}
|
|
|
|
// sqliteDSN keeps the store's two pools on one database and lets a writer that
|
|
// meets the other pool's reader wait for it instead of failing outright. A bare
|
|
// :memory: is private to each connection, so it has to be named and shared.
|
|
func sqliteDSN(dbFile string) string {
|
|
dsn := dbFile
|
|
if dsn == ":memory:" {
|
|
dsn = fmt.Sprintf("file:seaweedfs%d?mode=memory&cache=shared", time.Now().UnixNano())
|
|
}
|
|
separator := "?"
|
|
if strings.Contains(dsn, "?") {
|
|
separator = "&"
|
|
}
|
|
return dsn + separator + "_pragma=busy_timeout(10000)"
|
|
}
|
|
|
|
func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (err error) {
|
|
|
|
store.SupportBucketTable = true
|
|
store.SqlGenerator = &mysql.SqlGenMysql{
|
|
CreateTableSqlTemplate: createTable,
|
|
DropTableSqlTemplate: "drop table if exists `%s`",
|
|
UpsertQueryTemplate: upsertQuery,
|
|
}
|
|
|
|
dsn := sqliteDSN(dbFile)
|
|
|
|
db, dbErr := sql.Open("sqlite", dsn)
|
|
if dbErr != nil {
|
|
if db != nil {
|
|
db.Close()
|
|
}
|
|
return fmt.Errorf("can not connect to %s error:%v", dbFile, dbErr)
|
|
}
|
|
|
|
if err = db.Ping(); err != nil {
|
|
return fmt.Errorf("connect to %s error:%v", dbFile, err)
|
|
}
|
|
|
|
if err = store.UseConnectionPools(db, func() (*sql.DB, error) {
|
|
return sql.Open("sqlite", dsn)
|
|
}, 1, 1, 0); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {
|
|
return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)
|
|
}
|
|
|
|
return nil
|
|
}
|