mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-17 03:50:51 +02:00
87332eb60b
* pb: add multipart concurrency fields to RemoteConf and tier move requests RemoteConf gains upload_concurrency/download_concurrency (0 = client default); VolumeTierMoveDatToRemote/FromRemote requests gain a concurrency field (0 = backend default). * remote storage: honor RemoteConf upload/download concurrency in s3 and azure clients s3 client: ReadFile passes conf download_concurrency to the downloader, WriteFile uses upload_concurrency for the uploader; previously hard-coded 1 upload / 5 download parts. 0 keeps defaults. Same for azure client. * storage: plumb concurrency through backend interface and tier upload/download BackendStorage.CopyFile/DownloadFile take a concurrency hint (<=0 = backend configured default); s3 backend reads upload_concurrency/download_concurrency from scaffold config with parseConcurrency fallback, rclone updated to the new signature. Tier move gRPC handlers forward the request concurrency to the backend. * shell: -upload_concurrency/-download_concurrency for remote.configure, -concurrent for volume.tier remote.configure exposes upload/download concurrency persisted into RemoteConf; volume.tier move/evict commands forward -concurrent to the tier move requests. Documented in master-cloud.toml scaffold. * test: cover concurrency propagation in remote tier integration test * remote.configure: merge existing config on partial update Load the stored RemoteConf before saving so a partial update (e.g. only -upload_concurrency) preserves credentials, endpoints, and type instead of replacing them with new-config defaults. Only treat a confirmed ErrNotFound as a new configuration; propagate all other load errors so a transient filer failure does not overwrite stored settings. On a type transition, reset backend-specific fields to the destination type's new-config defaults rather than inheriting the old backend's empty values. Bound configured concurrency to a sane maximum. * remote storage: honor configured download concurrency in S3 and Azure ReadFileWithConcurrency now resolves a zero request override against the client's configured download_concurrency (new downloadConcurrency() helpers), so the remote-mount/cache read path honors RemoteConf.DownloadConcurrency instead of the hard-coded default. Azure also clamps the resolved value to math.MaxUint16 regardless of whether the fallback was used, preventing uint16 wraparound when a configured value exceeds 65535. * shell: rename -concurrent to -concurrency and validate tier transfer bounds Rename the -concurrent flag to -concurrency across volume.tier.upload, volume.tier.download, and volume.tier.compact to match the proto field and RemoteConf field names. Add validateTierConcurrency to reject values that would wrap int32 or exceed a 1024 cap before constructing the request. * server: clamp tier move concurrency in gRPC handlers Add clampTierConcurrency to both VolumeTierMoveDatToRemote and VolumeTierMoveDatFromRemote handlers so a direct gRPC caller cannot spawn an unbounded number of network workers. * trim verbose comments added with concurrency feature Remove redundant doc comments on the backend interface, rclone backend, s3_backend parseConcurrency, and test helpers that restated the obvious. * remote.configure: apply type defaults before re-parse so explicit flags win applyTypeDefaults ran after the second flag parse, overwriting explicit destination flags (e.g. -s3.region=eu-west-1) with new-config defaults. Move the type-transition default reset before the re-parse so user-supplied flags override the destination defaults. * remote.configure: only treat explicit -type as a type transition The first parse defaults -type to s3, so a concurrency-only update on an existing non-S3 config captured requestedType=s3 and wrongly triggered a type transition, resetting the stored backend to S3. Use fs.Visit to detect whether -type was explicitly supplied; an omitted -type keeps the stored backend. --------- Co-authored-by: Jack Meredith <9480542+jackusm@users.noreply.github.com> Co-authored-by: Chris Lu <chris.lu@gmail.com>
35 lines
1.3 KiB
TOML
35 lines
1.3 KiB
TOML
|
|
# Put this file to one of the location, with descending priority
|
|
# ./master.toml
|
|
# $HOME/.seaweedfs/master.toml
|
|
# /etc/seaweedfs/master.toml
|
|
# this file is read by master
|
|
|
|
[master.maintenance]
|
|
# periodically run these scripts are the same as running them from 'weed shell'
|
|
# Scripts are skipped while an admin server is connected.
|
|
scripts = """
|
|
lock
|
|
ec.encode -fullPercent=95 -quietFor=1h
|
|
ec.rebuild -apply
|
|
ec.balance -apply
|
|
volume.balance -apply
|
|
volume.fix.replication -apply
|
|
unlock
|
|
"""
|
|
sleep_minutes = 17 # sleep minutes between each script execution
|
|
|
|
# configurations for tiered cloud storage
|
|
# old volumes are transparently moved to cloud for cost efficiency
|
|
[storage.backend]
|
|
[storage.backend.s3.default]
|
|
enabled = true
|
|
aws_access_key_id = "any" # if empty, loads from the shared credentials file (~/.aws/credentials).
|
|
aws_secret_access_key = "any" # if empty, loads from the shared credentials file (~/.aws/credentials).
|
|
region = "us-east-2"
|
|
bucket = "volume_bucket" # an existing bucket
|
|
endpoint = "http://server2:8333"
|
|
storage_class = "STANDARD_IA"
|
|
# upload_concurrency = 5 # concurrent multipart part uploads per volume (volume.tier.upload -concurrent overrides)
|
|
# download_concurrency = 5 # concurrent multipart part downloads per volume (volume.tier.download -concurrent overrides)
|