From 241541c0266a0e08c50cc9fd2cd9a3ea1dbb0ecd Mon Sep 17 00:00:00 2001 From: Carlos Leyva <66577002+carlos-leyva-guerrero@users.noreply.github.com> Date: Thu, 3 Sep 2026 07:11:44 +0200 Subject: [PATCH] filer: SQL store pool defaults that survive a concurrent walk (#11110) * filer: SQL store pool defaults survive concurrent walks (idle == open == 50, lifetime 300s) The code defaults for the four SQL stores were connection_max_idle=2 with NO default for connection_max_open (unlimited) or lifetime, while the scaffold filer.toml documents 10/50/300 -- so an env-configured or minimal-toml filer got the worst possible pool. Under a concurrent listing burst (s3.lifecycle.run-shard walks 16 shards in parallel) every operation released above the 2 idle slots closes its TCP connection, so the walk opens a fresh connection per operation until the filer exhausts its ephemeral ports: list /buckets/... : failed to connect ... dial tcp ...:5432: connect: cannot assign requested address Measured on a production filer: 0 -> 28k TIME_WAIT with only ~1.3k concurrent, and in the minimal docker-compose reproduction (2000-dir bucket, port range narrowed to 400): the whole range in TIME_WAIT with only ~12 ESTABLISHED. Default all three knobs, with idle == open so released connections are kept and reused: idle connections only accumulate up to the actual peak concurrency and connection_max_lifetime_seconds recycles them, so a quiet deployment holds nothing extra. An explicit 0 still disables the caps as before. The scaffold's connection_max_idle moves 10 -> 50 to match. With this change the same reproduction completes all 16 shards with the default configuration (TIME_WAIT peak 19 vs the whole port range). * filer: trim the SQL pool default comments One line of the non-obvious why is enough; the rest narrated the code. Claude-Session: https://claude.ai/code/session_018DWwctzD4T2DmnczPRM47t * filer: leave the SQL stores' connection_max_open unset A listing holds its connection for the whole row iteration while its callback runs another query -- FilerStoreWrapper.maybeReadHardLink does a KvGet per hard-linked entry -- so every concurrent listing needs two connections from the same pool. With a default cap, listings past the cap wedge: 60 concurrent listings over hard-linked entries made no progress at all against a 50 connection pool, and the wrapper's context.WithoutCancel leaves the waiters without a deadline. The idle pool is what fixes the connection churn: idle 50 with an unbounded max_open holds the same 14 postgres sessions across a 16-way listing burst that opened 455 with idle 2. Claude-Session: https://claude.ai/code/session_018DWwctzD4T2DmnczPRM47t --------- Co-authored-by: Chris Lu --- weed/command/scaffold/filer.toml | 8 ++++---- weed/filer/mysql/mysql_store.go | 8 ++++++-- weed/filer/mysql2/mysql2_store.go | 8 ++++++-- weed/filer/postgres/postgres_store.go | 8 ++++++-- weed/filer/postgres2/postgres2_store.go | 8 ++++++-- 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 52df80744..397b93887 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -68,7 +68,7 @@ port = 3306 username = "root" password = "" database = "" # create or use an existing database -connection_max_idle = 10 +connection_max_idle = 50 connection_max_open = 50 connection_max_lifetime_seconds = 300 interpolateParams = false @@ -97,7 +97,7 @@ port = 3306 username = "root" password = "" database = "" # create or use an existing database -connection_max_idle = 10 +connection_max_idle = 50 connection_max_open = 50 connection_max_lifetime_seconds = 300 interpolateParams = false @@ -127,7 +127,7 @@ sslmode = "disable" # sslkey = "/path/to/client.key" # client private key file # sslrootcert = "/path/to/ca.crt" # CA certificate file # sslcrl = "/path/to/client.crl" # Certificate Revocation List (CRL) (optional) -connection_max_idle = 10 +connection_max_idle = 50 connection_max_open = 50 connection_max_lifetime_seconds = 300 # Set to true when using PgBouncer connection pooler @@ -166,7 +166,7 @@ sslmode = "disable" # sslkey = "/path/to/client.key" # client private key file # sslrootcert = "/path/to/ca.crt" # CA certificate file # sslcrl = "/path/to/client.crl" # Certificate Revocation List (CRL) (optional) -connection_max_idle = 10 +connection_max_idle = 50 connection_max_open = 50 connection_max_lifetime_seconds = 300 # Set to true when using PgBouncer connection pooler diff --git a/weed/filer/mysql/mysql_store.go b/weed/filer/mysql/mysql_store.go index abcef6fe6..5a7cf9da0 100644 --- a/weed/filer/mysql/mysql_store.go +++ b/weed/filer/mysql/mysql_store.go @@ -33,8 +33,12 @@ func (store *MysqlStore) GetName() string { } func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) { - // Absent key keeps a pooled default; an explicit 0 disables the idle pool. - configuration.SetDefault(prefix+"connection_max_idle", 2) + // Fewer idle slots than concurrent operations means a fresh connection per + // operation, until the filer runs out of ephemeral ports. connection_max_open + // stays unset: a listing runs a second query from its own callback, so a + // bounded pool deadlocks once the concurrency reaches it. + configuration.SetDefault(prefix+"connection_max_idle", 50) + configuration.SetDefault(prefix+"connection_max_lifetime_seconds", 300) // Default on so minimal configs avoid the duplicate-key roundtrip the // inode-index KvPut would otherwise emit on every write. configuration.SetDefault(prefix+"enableUpsert", true) diff --git a/weed/filer/mysql2/mysql2_store.go b/weed/filer/mysql2/mysql2_store.go index 0783398a3..e8b0af261 100644 --- a/weed/filer/mysql2/mysql2_store.go +++ b/weed/filer/mysql2/mysql2_store.go @@ -33,8 +33,12 @@ func (store *MysqlStore2) GetName() string { } func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) { - // Absent key keeps a pooled default; an explicit 0 disables the idle pool. - configuration.SetDefault(prefix+"connection_max_idle", 2) + // Fewer idle slots than concurrent operations means a fresh connection per + // operation, until the filer runs out of ephemeral ports. connection_max_open + // stays unset: a listing runs a second query from its own callback, so a + // bounded pool deadlocks once the concurrency reaches it. + configuration.SetDefault(prefix+"connection_max_idle", 50) + configuration.SetDefault(prefix+"connection_max_lifetime_seconds", 300) // Default on so minimal configs avoid the duplicate-key roundtrip the // inode-index KvPut would otherwise emit on every write. configuration.SetDefault(prefix+"enableUpsert", true) diff --git a/weed/filer/postgres/postgres_store.go b/weed/filer/postgres/postgres_store.go index c694f77c3..62f27b327 100644 --- a/weed/filer/postgres/postgres_store.go +++ b/weed/filer/postgres/postgres_store.go @@ -28,8 +28,12 @@ func (store *PostgresStore) GetName() string { } func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) { - // Absent key keeps a pooled default; an explicit 0 disables the idle pool. - configuration.SetDefault(prefix+"connection_max_idle", 2) + // Fewer idle slots than concurrent operations means a fresh connection per + // operation, until the filer runs out of ephemeral ports. connection_max_open + // stays unset: a listing runs a second query from its own callback, so a + // bounded pool deadlocks once the concurrency reaches it. + configuration.SetDefault(prefix+"connection_max_idle", 50) + configuration.SetDefault(prefix+"connection_max_lifetime_seconds", 300) // Default on so minimal configs are not exposed to duplicate-key tx // poisoning on Postgres; an explicit false still disables it. configuration.SetDefault(prefix+"enableUpsert", true) diff --git a/weed/filer/postgres2/postgres2_store.go b/weed/filer/postgres2/postgres2_store.go index e846bdf9f..2eb85764a 100644 --- a/weed/filer/postgres2/postgres2_store.go +++ b/weed/filer/postgres2/postgres2_store.go @@ -33,8 +33,12 @@ func (store *PostgresStore2) GetName() string { } func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix string) (err error) { - // Absent key keeps a pooled default; an explicit 0 disables the idle pool. - configuration.SetDefault(prefix+"connection_max_idle", 2) + // Fewer idle slots than concurrent operations means a fresh connection per + // operation, until the filer runs out of ephemeral ports. connection_max_open + // stays unset: a listing runs a second query from its own callback, so a + // bounded pool deadlocks once the concurrency reaches it. + configuration.SetDefault(prefix+"connection_max_idle", 50) + configuration.SetDefault(prefix+"connection_max_lifetime_seconds", 300) // Default on so minimal configs are not exposed to duplicate-key tx // poisoning on Postgres; an explicit false still disables it. configuration.SetDefault(prefix+"enableUpsert", true)