Github-Mirror/seaweedfs
1
0 Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-10-23 22:40:39 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
0b197867a1671c1e0136244eeacf2ead4a5200a1
seaweedfs/weed/command/scaffold/filer.toml
T
Chris LuandDevin 0b197867a1 filer leveldb2: tunable cache, buffer, block and table sizes (#11927) 
* filer leveldb2: read cache and buffer sizes from filer.toml

The leveldb2 store hardcoded its block cache (32MiB) and write buffer
(16MiB) and offered no way to tune the table block size or compaction
table size. Optional block_cache_size_mb, write_buffer_size_mb,
block_size_kb and compaction_table_size_mb keys under [leveldb2] now
override them; zero or negative values keep the previous defaults.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* filer leveldb2: document tuning options in scaffold filer.toml

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* filer leveldb2: saturate size options instead of overflowing int

The option fields take int, so a MiB config value past math.MaxInt/MiB
(e.g. 2048 on 32-bit) wrapped negative and goleveldb would panic sizing
the memtable. Clamp the product instead of the input.

* filer leveldb2: reject size options too large for int bytes

Clamping a MiB setting at math.MaxInt silently hands goleveldb a
different size than configured; fail Initialize on values that cannot
fit instead.

* filer leveldb3: same tunable cache, buffer, block and table sizes

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* filer leveldb2/3: exercise oversized-option rejection on 64-bit too

The tests skipped whenever int was 64-bit, but write_buffer_size_mb
values above MaxInt/1MiB still overflow there, so the rejection path
had no coverage on the primary platform. Use MaxInt/1MiB + 1 so the
rejection triggers on every platform.

Generated with [Devin](https://devin.ai)

Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-21 20:39:06 -07:00

503 lines
17 KiB
TOML
Raw Blame History

# A sample TOML config file for SeaweedFS filer store
# Used with "weed filer" or "weed server -filer"
# Put this file to one of the location, with descending priority
# ./filer.toml
# $HOME/.seaweedfs/filer.toml
# /etc/seaweedfs/filer.toml
####################################################
# Customizable filer server options
####################################################
[filer.options]
# with http DELETE, by default the filer would check whether a folder is empty.
# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
recursive_delete = false
#max_file_name_length = 255
# for S3: how long to wait before deleting an empty folder.
# increase this if using tools like Spark that create temporary directories.
#s3.empty_folder_cleanup_delay = "2m"
# where the filer's internal system metadata log chunks are stored.
# By default the log follows the filer's -collection flag; set this to keep
# the internal log in its own collection, out of the default one.
# Keep the target stable: log chunks written under an older value are not
# moved, and old log entries keep referencing that collection's volumes.
#metaLog.collection = "filer-meta"
#metaLog.replication = ""
####################################################
# The following are filer store options
####################################################
[leveldb2]
# local on disk, mostly for simple single-machine setup, fairly scalable
# faster than previous leveldb, recommended.
enabled = true
dir = "./filerldb2" # directory to store level db files
# optional tuning, applied to each of the 8 partitioned stores
# block_cache_size_mb = 32 # sorted-table read cache
# write_buffer_size_mb = 16 # memtable size before flushing to disk
# block_size_kb = 4 # uncompressed block ('page') size inside table files
# compaction_table_size_mb = 2 # target size of generated table files
[leveldb3]
# similar to leveldb2.
# each bucket has its own meta store.
enabled = false
dir = "./filerldb3" # directory to store level db files
# optional tuning, applied to the default store and each bucket store
# block_cache_size_mb = 32 # sorted-table read cache
# write_buffer_size_mb = 16 # memtable size before flushing to disk
# block_size_kb = 4 # uncompressed block ('page') size inside table files
# compaction_table_size_mb = 2 # target size of generated table files
[rocksdb]
# local on disk, similar to leveldb
# since it is using a C wrapper, you need to install rocksdb and build it by yourself
enabled = false
dir = "./filerrdb" # directory to store rocksdb files
[sqlite]
# local on disk, similar to leveldb
enabled = false
dbFile = "./filer.db" # sqlite db file
[mysql] # or memsql, tidb
# CREATE TABLE IF NOT EXISTS `filemeta` (
# `dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
# `name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
# `directory` TEXT NOT NULL COMMENT 'full path to parent directory',
# `meta` LONGBLOB,
# PRIMARY KEY (`dirhash`, `name`)
# ) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
enabled = false
# dsn will take priority over "hostname, port, username, password, database".
# [username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
dsn = "root@tcp(localhost:3306)/seaweedfs?collation=utf8mb4_bin"
enable_tls = false
ca_crt = "" # path to CA cert (PEM) — optional; if empty, the system trust store is used
client_crt = "" # path to client cert (PEM) — only when server requires mTLS; must be set together with client_key
client_key = "" # path to client key (PEM) — only when server requires mTLS; must be set together with client_crt
tls_insecure_skip_verify = false # skip server cert verification (use only for testing or with self-signed certs)
tls_server_name = "" # override SNI / cert hostname; leave empty to use `hostname` above
hostname = "localhost"
port = 3306
username = "root"
password = ""
database = "" # create or use an existing database
connection_max_idle = 50
connection_max_open = 50
connection_max_lifetime_seconds = 300
interpolateParams = false
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
enableUpsert = true
# Default uses the row-alias form (`AS new`) added in MySQL 8.0.19 and is the
# preferred syntax there. For MariaDB (any version) and MySQL 5.7, override
# with the form below — MariaDB does not support row aliases in
# INSERT ... ON DUPLICATE KEY UPDATE:
# upsertQuery = """INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES (?,?,?,?) ON DUPLICATE KEY UPDATE `meta` = VALUES(`meta`)"""
upsertQuery = """INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES (?,?,?,?) AS `new` ON DUPLICATE KEY UPDATE `meta` = `new`.`meta`"""
[mysql2] # or memsql, tidb
enabled = false
createTable = """
CREATE TABLE IF NOT EXISTS `%s` (
`dirhash` BIGINT NOT NULL,
`name` VARCHAR(766) NOT NULL,
`directory` TEXT NOT NULL,
`meta` LONGBLOB,
PRIMARY KEY (`dirhash`, `name`)
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
"""
hostname = "localhost"
port = 3306
username = "root"
password = ""
database = "" # create or use an existing database
connection_max_idle = 50
connection_max_open = 50
connection_max_lifetime_seconds = 300
interpolateParams = false
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
enableUpsert = true
upsertQuery = """INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES (?,?,?,?) AS `new` ON DUPLICATE KEY UPDATE `meta` = `new`.`meta`"""
[postgres] # or cockroachdb, YugabyteDB
# createTable = true # auto-create filemeta with the default schema
# createTable = false # skip table creation (default; for restricted DB roles)
# createTable = """CREATE TABLE IF NOT EXISTS "%s" (...)""" # custom template
enabled = false
hostname = "localhost"
port = 5432
username = "postgres"
password = ""
database = "postgres" # create or use an existing database
schema = ""
sslmode = "disable"
# SSL certificate options for secure connections
# For sslmode=verify-full, uncomment and configure the following:
# sslcert = "/path/to/client.crt" # client certificate file
# sslkey = "/path/to/client.key" # client private key file
# sslrootcert = "/path/to/ca.crt" # CA certificate file
# sslcrl = "/path/to/client.crl" # Certificate Revocation List (CRL) (optional)
connection_max_idle = 50
connection_max_open = 50
connection_max_lifetime_seconds = 300
# Set to true when using PgBouncer connection pooler
pgbouncer_compatible = false
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
enableUpsert = true
upsertQuery = """
INSERT INTO "%[1]s" (dirhash, name, directory, meta)
VALUES($1, $2, $3, $4)
ON CONFLICT (dirhash, name) DO UPDATE SET
directory=EXCLUDED.directory,
meta=EXCLUDED.meta
"""
[postgres2]
enabled = false
createTable = """
CREATE TABLE IF NOT EXISTS "%s" (
dirhash BIGINT,
name VARCHAR(65535),
directory VARCHAR(65535),
meta bytea,
PRIMARY KEY (dirhash, name)
);
"""
hostname = "localhost"
port = 5432
username = "postgres"
password = ""
database = "postgres" # create or use an existing database
schema = ""
sslmode = "disable"
# SSL certificate options for secure connections
# For sslmode=verify-full, uncomment and configure the following:
# sslcert = "/path/to/client.crt" # client certificate file
# sslkey = "/path/to/client.key" # client private key file
# sslrootcert = "/path/to/ca.crt" # CA certificate file
# sslcrl = "/path/to/client.crl" # Certificate Revocation List (CRL) (optional)
connection_max_idle = 50
connection_max_open = 50
connection_max_lifetime_seconds = 300
# Set to true when using PgBouncer connection pooler
pgbouncer_compatible = false
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
enableUpsert = true
upsertQuery = """
INSERT INTO "%[1]s" (dirhash, name, directory, meta)
VALUES($1, $2, $3, $4)
ON CONFLICT (dirhash, name) DO UPDATE SET
directory=EXCLUDED.directory,
meta=EXCLUDED.meta
"""
[cassandra2]
# CREATE TABLE filemeta (
# dirhash bigint,
# directory varchar,
# name varchar,
# meta blob,
# PRIMARY KEY ((dirhash, directory), name)
# ) WITH CLUSTERING ORDER BY (name ASC);
enabled = false
keyspace = "seaweedfs"
hosts = [
"localhost:9042",
]
username = ""
password = ""
# Set the CA certificate path
ssl_ca_path = ""
# Set the client certificate path
ssl_cert_path = ""
# Set the client private key path
ssl_key_path = ""
# Check host name in the certificate
ssl_enable_host_verification = true
# This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
superLargeDirectories = []
# Name of the datacenter local to this filer, used as host selection fallback.
localDC = ""
# Gocql connection timeout, default: 600ms
connection_timeout_millisecond = 600
[hbase]
enabled = false
zkquorum = ""
table = "seaweedfs"
[redis2]
enabled = false
address = "localhost:6379"
username = ""
password = ""
database = 0
# prefix for filer redis keys
keyPrefix = ""
enable_tls = false
ca_cert_path = ""
client_cert_path = ""
client_key_path = ""
# This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
superLargeDirectories = []
# Connection tuning, accepted by every redis filer store section. Each key keeps the go-redis
# default when left at 0, and a timeout cannot be turned off - a filer that never times out a read
# is what stalls a failover. Shorter read timeouts make the filer notice a dead redis sooner, which
# matters most for the sentinel store: every pooled connection to the old master has to time out
# before a failover is absorbed.
# max_retries = 0 # 0 is three retries, -1 disables retrying
# min_retry_backoff_millisecond = 0 # default 8
# max_retry_backoff_millisecond = 0 # default 512
# dial_timeout_millisecond = 0 # default 5000
# read_timeout_millisecond = 0 # default 3000
# write_timeout_millisecond = 0 # default 3000, follows read_timeout_millisecond
# pool_size = 0 # default 10 per CPU
# pool_timeout_millisecond = 0 # default read_timeout_millisecond + 1000
# min_idle_conns = 0
# max_idle_conns = 0
# conn_max_idle_time_seconds = 0 # default 1800
# conn_max_lifetime_seconds = 0 # default unlimited
[redis2_sentinel]
enabled = false
addresses = ["172.22.12.7:26379","172.22.12.8:26379","172.22.12.9:26379"]
masterName = "master"
username = ""
password = ""
# credentials used to authenticate against the sentinel nodes themselves (if they require auth)
sentinel_username = ""
sentinel_password = ""
database = 0
# prefix for filer redis keys
keyPrefix = ""
enable_tls = false
ca_cert_path = ""
client_cert_path = ""
client_key_path = ""
# see [redis2] for the connection tuning keys
[redis_cluster2]
enabled = false
addresses = [
"localhost:30001",
"localhost:30002",
"localhost:30003",
"localhost:30004",
"localhost:30005",
"localhost:30006",
]
username = ""
password = ""
# prefix for filer redis keys
keyPrefix = ""
enable_tls = false
ca_cert_path = ""
client_cert_path = ""
client_key_path = ""
# allows reads from slave servers or the master, but all writes still go to the master
useReadOnly = false
# automatically use the closest Redis server for reads
routeByLatency = false
# This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
superLargeDirectories = []
# The following lua redis stores uses lua to ensure atomicity
[redis_lua]
enabled = false
address = "localhost:6379"
username = ""
password = ""
database = 0
# prefix for filer redis keys
keyPrefix = ""
enable_tls = false
ca_cert_path = ""
client_cert_path = ""
client_key_path = ""
# This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
superLargeDirectories = []
[redis_lua_sentinel]
enabled = false
addresses = ["172.22.12.7:26379","172.22.12.8:26379","172.22.12.9:26379"]
masterName = "master"
username = ""
password = ""
database = 0
# prefix for filer redis keys
keyPrefix = ""
enable_tls = false
ca_cert_path = ""
client_cert_path = ""
client_key_path = ""
[redis_lua_cluster]
enabled = false
addresses = [
"localhost:30001",
"localhost:30002",
"localhost:30003",
"localhost:30004",
"localhost:30005",
"localhost:30006",
]
username = ""
password = ""
# prefix for filer redis keys
keyPrefix = ""
enable_tls = false
ca_cert_path = ""
client_cert_path = ""
client_key_path = ""
# allows reads from slave servers or the master, but all writes still go to the master
readOnly = false
# automatically use the closest Redis server for reads
routeByLatency = false
# This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
superLargeDirectories = []
[etcd]
enabled = false
servers = "localhost:2379"
username = ""
password = ""
key_prefix = "seaweedfs."
timeout = "3s"
# Set the CA certificate path
tls_ca_file=""
# Set the client certificate path
tls_client_crt_file=""
# Set the client private key path
tls_client_key_file=""
[mongodb]
enabled = false
uri = "mongodb://localhost:27017"
username = ""
password = ""
ssl = false
ssl_ca_file = ""
ssl_cert_file = ""
ssl_key_file = ""
insecure_skip_verify = false
option_pool_size = 0
database = "seaweedfs"
[elastic7]
enabled = false
servers = [
"http://localhost1:9200",
"http://localhost2:9200",
"http://localhost3:9200",
]
username = ""
password = ""
sniff_enabled = false
healthcheck_enabled = false
# increase the value is recommend, be sure the value in Elastic is greater or equal here
index.max_result_window = 10000
# for Elasticsearch 8.x clusters
[elastic8]
enabled = false
servers = [
"http://localhost1:9200",
"http://localhost2:9200",
"http://localhost3:9200",
]
username = ""
password = ""
sniff_enabled = false
healthcheck_enabled = false
# increase the value is recommend, be sure the value in Elastic is greater or equal here
index.max_result_window = 10000
[arangodb] # in development dont use it
enabled = false
db_name = "seaweedfs"
servers=["http://localhost:8529"] # list of servers to connect to
# only basic auth supported for now
username=""
password=""
# skip tls cert validation
insecure_skip_verify = true
[ydb] # https://ydb.tech/
enabled = false
dsn = "grpc://localhost:2136?database=/local"
prefix = "seaweedfs"
useBucketPrefix = true # Fast Bucket Deletion
poolSizeLimit = 50
dialTimeOut = 10
# Authenticate produced with one of next environment variables:
# YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=<path/to/sa_key_file> — used service account key file by path
# YDB_ANONYMOUS_CREDENTIALS="1" — used for authenticate with anonymous access. Anonymous access needs for connect to testing YDB installation
# YDB_METADATA_CREDENTIALS="1" — used metadata service for authenticate to YDB from yandex cloud virtual machine or from yandex function
# YDB_ACCESS_TOKEN_CREDENTIALS=<access_token> — used for authenticate to YDB with short-life access token. For example, access token may be IAM token
##########################
##########################
# To add path-specific filer store:
#
# 1. Add a name following the store type separated by a dot ".". E.g., cassandra2.tmp
# 2. Add a location configuration. E.g., location = "/tmp/"
# 3. Copy and customize all other configurations.
# Make sure they are not the same if using the same store type!
# 4. Set enabled to true
#
# The following is just using redis as an example
##########################
[redis2.tmp]
enabled = false
location = "/tmp/"
address = "localhost:6379"
username = ""
password = ""
database = 1
keyPrefix = ""
[tikv]
enabled = false
# If you have many pd address, use ',' split then:
# pdaddrs = "pdhost1:2379, pdhost2:2379, pdhost3:2379"
pdaddrs = "localhost:2379"
# prefix for filer TiKV keys, useful for sharing a TiKV cluster with multiple seaweedfs clusters
keyPrefix = ""
# Enable 1PC
enable_1pc = false
# batch delete count, default 10000 in code
#batchdelete_count = 20000
# Set the CA certificate path
ca_path=""
# Set the certificate path
cert_path=""
# Set the private key path
key_path=""
# The name list used to verify the cn name
verify_cn=""
[foundationdb]
# FoundationDB provides ACID transactions and horizontal scalability.
# Requires: go build -tags foundationdb
enabled = false
cluster_file = "/etc/foundationdb/fdb.cluster"
# api_version = 740
# timeout = "5s"
# directory_prefix = "seaweedfs"
# For bulk ingestion, enable batching: batch_enabled = true
[tarantool]
address = "localhost:3301"
user = "guest"
password = ""
timeout = "5s"
maxReconnects = 1000
Reference in new issue View Git Blame Copy Permalink