mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-22 22:11:39 +02:00
Javier de la TorreandJavier de la Torre
21add33b6e
* s3 backend: optional HTTP client settings for the cloud tier The S3 tier backend builds its client from a bare aws.Config, so it always uses Go's default transport: HTTP/2 wherever the endpoint offers it over TLS, no cap on connections per host, and a 90s idle timeout. Against some S3-compatible services that is a poor fit. Over HTTP/2 every upload and range read of a tiered volume shares one connection and its flow-control window; with a small window, tier uploads and concurrent reads of tiered volumes become throughput-bound on that one connection, while HTTP/1.1 opens a connection per concurrent request. Some services also limit parallel connections per client address, or drop idle connections before 90s. Add three optional settings to [storage.backend.s3.<id>], all off by default so existing deployments keep the SDK's default client: - disable_http2: keep the client on HTTP/1.1 (ForceAttemptHTTP2=false and an empty TLSNextProto, the documented opt-out); - max_connections: MaxConnsPerHost, 0 = no cap; - idle_connection_timeout_seconds: IdleConnTimeout, 0 = Go's default. The session cache key includes the options when any is set, so two backends on the same endpoint with different settings do not share a client. ToProperties reports the settings; the master.toml scaffold documents them. Tests: TestS3BackendHTTPOptionsReachTheClient fails without the change (the settings never reach the client); TestS3BackendDefaultHTTPClientUnchanged, TestS3BackendSessionsDifferByHTTPOptions, TestParseNonNegative. * s3 backend: reject an idle timeout too large for time.Duration An idle_connection_timeout_seconds above math.MaxInt64/1e9 overflowed when multiplied by time.Second. The negative result made the transport keep its default timeout and ToProperties export a negative value. Warn and use the default instead. * s3 backend: parse the idle timeout as 64-bit seconds parseIdleConnTimeout went through strconv.Atoi, so on 32-bit platforms any value above 2147483647 seconds was rejected while the same value was accepted on 64-bit, and the overflow test failed under GOARCH=386. Parse with ParseInt(…, 64) and export the effective value with FormatInt so the accepted range is the same on every platform. --------- Co-authored-by: Javier de la Torre <jatorre@lakehousebox.com>
58 lines
2.6 KiB
TOML
58 lines
2.6 KiB
TOML
# Put this file to one of the location, with descending priority
|
|
# ./master.toml
|
|
# $HOME/.seaweedfs/master.toml
|
|
# /etc/seaweedfs/master.toml
|
|
# this file is read by master
|
|
|
|
[master.maintenance]
|
|
# periodically run these scripts are the same as running them from 'weed shell'
|
|
# Scripts are skipped while an admin server is connected.
|
|
scripts = """{{DEFAULT_MAINTENANCE_SCRIPTS}}"""
|
|
sleep_minutes = 17 # sleep minutes between each script execution
|
|
|
|
|
|
[master.sequencer]
|
|
type = "raft" # Choose [raft|snowflake] type for storing the file id sequence
|
|
# when sequencer.type = snowflake, the snowflake id must be different from other masters
|
|
sequencer_snowflake_id = 0 # any number between 1~1023
|
|
|
|
|
|
# configurations for tiered cloud storage
|
|
# old volumes are transparently moved to cloud for cost efficiency
|
|
[storage.backend]
|
|
[storage.backend.s3.default]
|
|
enabled = false
|
|
aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
|
|
aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
|
|
region = "us-east-2"
|
|
bucket = "your_bucket_name" # an existing bucket
|
|
endpoint = ""
|
|
storage_class = "STANDARD_IA"
|
|
# optional HTTP client tuning for S3-compatible services:
|
|
# disable_http2 = false # true keeps the client on HTTP/1.1, one connection per concurrent request
|
|
# max_connections = 0 # cap on connections to the endpoint, 0 = no cap
|
|
# idle_connection_timeout_seconds = 0 # close idle connections sooner, 0 = Go default (90s)
|
|
|
|
# create this number of logical volumes if no more writable volumes
|
|
# count_x means how many copies of data.
|
|
# e.g.:
|
|
# 000 has only one copy, copy_1
|
|
# 010 and 001 has two copies, copy_2
|
|
# 011 has only 3 copies, copy_3
|
|
[master.volume_growth]
|
|
copy_1 = 7 # create 1 x 7 = 7 actual volumes
|
|
copy_2 = 6 # create 2 x 6 = 12 actual volumes
|
|
copy_3 = 3 # create 3 x 3 = 9 actual volumes
|
|
copy_other = 1 # create n x 1 = n actual volumes
|
|
threshold = 0.9 # create threshold
|
|
reservation_timeout = "5m"# capacity reservation timeout before unreleased reservations expire
|
|
disable = false # disables volume growth if true
|
|
|
|
# configuration flags for replication
|
|
[master.replication]
|
|
# any replication counts should be considered minimums. If you specify 010 and
|
|
# have 3 different racks, that's still considered writable. Writes will still
|
|
# try to replicate to all available volumes. You should only use this option
|
|
# if you are doing your own replication or periodic sync of volumes.
|
|
treat_replication_as_minimums = false
|