Files
seaweedfs/weed/admin/dash/config_toml.go
T
b2dd93eac8 admin: support [maintenance.s3_lifecycle] in admin.toml (#12661)
* admin: support [maintenance.s3_lifecycle] in admin.toml

The admin.toml overlay covers vacuum, volume_balance and erasure_coding but
not s3_lifecycle, so the bucket lifecycle job's settings can only be changed
from the web UI and are lost whenever the admin data directory does not
persist. That is the worst combination for this job type: its
meta_log_retention_days knob decides whether bucket rules ever apply to
objects written before the job was enabled, and silently reverting it to the
default makes the rules no-op without any error.

Wire the four admin-side knobs into the existing overlay:

  meta_log_retention_days    retention window the meta-log replay can reach
  walker_interval_minutes    minimum gap between full bucket walks
  cluster_deletes_per_second cluster-wide delete budget
  cluster_deletes_burst      token-bucket burst for that budget

All four are admin config values, not worker values: the engine partition is
computed on the admin side and the delete budget is divided across
execute-capable workers at dispatch time. enabled/retry_limit/
retry_backoff_seconds come for free from the shared section handling.

Tested with TestApplyPluginConfigFromTomlS3Lifecycle, which also pins the
retention=0 case: 0 means unbounded and must survive as an explicit 0 rather
than being treated as unset.

* admin: fix meta_log_retention_days guidance in admin.toml scaffold

Both reviewers caught a real error. `0` is not "unbounded": runShard does

    retentionWindow := cfg.RetentionWindow
    if retentionWindow <= 0 {
        retentionWindow = maxTTL
    }

so zero falls back to the longest rule TTL, which keeps every replay rule in
replay and promotes nothing to the walk set. Recommending 0 to force full
scans was wrong, and so was the claim that pre-existing objects are only
reachable through the walk set -- the cold-start recovery walk covers them
regardless of the retention value.

Describe zero as "no retention info supplied" and tell operators to set the
window the filer meta-log actually retains. The code change is unaffected:
it only wires the four keys into the existing overlay.

* admin: tie s3_lifecycle walk-set scans to walker_interval_minutes in admin.toml

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* admin: show the s3_lifecycle retry defaults in admin.toml

The s3_lifecycle descriptor sets no retry limit or backoff, so a failed
run is not retried and the scheduler's 5 s backoff applies; the scaffold
listed the vacuum values instead.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* admin: shorten the s3_lifecycle toml section comment

The retention and walker knobs are read by the worker from the admin
values, not computed on the admin side.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* admin: cover cluster_deletes_burst in the s3_lifecycle toml test

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* admin: check the s3_lifecycle retention zero is a stored int64

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 13:46:14 -08:00

355 lines
12 KiB
Go

package dash
import (
"fmt"
"math"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
// TomlConfig is the subset of viper used to read admin.toml values.
type TomlConfig interface {
IsSet(key string) bool
GetBool(key string) bool
GetInt(key string) int
GetFloat64(key string) float64
GetString(key string) string
GetStringSlice(key string) []string
}
// ApplyMaintenanceConfigFromToml writes maintenance task settings declared in
// admin.toml through to the persisted task configs, so they survive data
// directory loss and override admin UI edits on restart. Absent keys keep
// their persisted values.
func (cp *ConfigPersistence) ApplyMaintenanceConfigFromToml(v TomlConfig) error {
var maintenanceConf *MaintenanceConfig
maintenanceChanged := false
if k := "maintenance.enabled"; v.IsSet(k) {
conf, err := cp.LoadMaintenanceConfig()
if err != nil {
return fmt.Errorf("load maintenance config: %w", err)
}
conf.Enabled = proto.Bool(v.GetBool(k))
// the policy lives in the per-task config files; don't snapshot it here
conf.Policy = nil
maintenanceConf = conf
maintenanceChanged = true
}
vacuumConf := vacuum.LoadConfigFromPersistence(cp)
vacuumChanged := applyBaseConfigFromToml(v, "maintenance.vacuum.", &vacuumConf.BaseConfig)
if k := "maintenance.vacuum.garbage_threshold"; v.IsSet(k) {
vacuumConf.GarbageThreshold = v.GetFloat64(k)
vacuumChanged = true
}
if k := "maintenance.vacuum.min_volume_age_seconds"; v.IsSet(k) {
vacuumConf.MinVolumeAgeSeconds = v.GetInt(k)
vacuumChanged = true
}
balanceConf := balance.LoadConfigFromPersistence(cp)
balanceChanged := applyBaseConfigFromToml(v, "maintenance.balance.", &balanceConf.BaseConfig)
if k := "maintenance.balance.imbalance_threshold"; v.IsSet(k) {
balanceConf.ImbalanceThreshold = v.GetFloat64(k)
balanceChanged = true
}
if k := "maintenance.balance.min_server_count"; v.IsSet(k) {
balanceConf.MinServerCount = v.GetInt(k)
balanceChanged = true
}
ecConf := erasure_coding.LoadConfigFromPersistence(cp)
ecChanged := applyBaseConfigFromToml(v, "maintenance.erasure_coding.", &ecConf.BaseConfig)
if k := "maintenance.erasure_coding.fullness_ratio"; v.IsSet(k) {
ecConf.FullnessRatio = v.GetFloat64(k)
ecChanged = true
}
if k := "maintenance.erasure_coding.quiet_for_seconds"; v.IsSet(k) {
ecConf.QuietForSeconds = v.GetInt(k)
ecChanged = true
}
if k := "maintenance.erasure_coding.min_size_mb"; v.IsSet(k) {
ecConf.MinSizeMB = v.GetInt(k)
ecChanged = true
}
if k := "maintenance.erasure_coding.collection_filter"; v.IsSet(k) {
ecConf.CollectionFilter = v.GetString(k)
ecChanged = true
}
if k := "maintenance.erasure_coding.preferred_tags"; v.IsSet(k) {
// viper does not split comma-separated values from env vars
var tags []string
for _, tag := range v.GetStringSlice(k) {
tags = append(tags, strings.Split(tag, ",")...)
}
ecConf.PreferredTags = util.NormalizeTagList(tags)
ecChanged = true
}
if k := "maintenance.erasure_coding.replica_placement"; v.IsSet(k) {
ecConf.ReplicaPlacement = v.GetString(k)
ecChanged = true
}
if k := "maintenance.erasure_coding.strict_placement"; v.IsSet(k) {
ecConf.StrictPlacement = v.GetBool(k)
ecChanged = true
}
if k := "maintenance.erasure_coding.free_slots_deadband"; v.IsSet(k) {
ecConf.FreeSlotsDeadband = v.GetInt(k)
if ecConf.FreeSlotsDeadband < 0 {
ecConf.FreeSlotsDeadband = 0
} else if ecConf.FreeSlotsDeadband > 100000 {
ecConf.FreeSlotsDeadband = 100000
}
ecChanged = true
}
if k := "maintenance.erasure_coding.max_shard_transfers"; v.IsSet(k) {
n := v.GetInt(k)
if n < 0 {
n = 0
} else if n > math.MaxInt32 {
n = math.MaxInt32
}
ecConf.MaxShardTransfers = &n
ecChanged = true
}
if !maintenanceChanged && !vacuumChanged && !balanceChanged && !ecChanged {
return nil
}
if !cp.IsConfigured() {
return fmt.Errorf("admin.toml maintenance settings require -dataDir to persist")
}
if maintenanceChanged {
if err := cp.SaveMaintenanceConfig(maintenanceConf); err != nil {
return fmt.Errorf("save maintenance config: %w", err)
}
glog.V(0).Infof("Applied [maintenance] settings from admin.toml (enabled: %v)", maintenanceConf.GetEnabled())
}
if vacuumChanged {
if err := cp.SaveVacuumTaskPolicy(vacuumConf.ToTaskPolicy()); err != nil {
return fmt.Errorf("save vacuum task config: %w", err)
}
glog.V(0).Infof("Applied [maintenance.vacuum] settings from admin.toml")
}
if balanceChanged {
if err := cp.SaveBalanceTaskPolicy(balanceConf.ToTaskPolicy()); err != nil {
return fmt.Errorf("save balance task config: %w", err)
}
glog.V(0).Infof("Applied [maintenance.balance] settings from admin.toml")
}
if ecChanged {
if err := cp.SaveErasureCodingTaskPolicy(ecConf.ToTaskPolicy()); err != nil {
return fmt.Errorf("save erasure coding task config: %w", err)
}
glog.V(0).Infof("Applied [maintenance.erasure_coding] settings from admin.toml")
}
return nil
}
func applyBaseConfigFromToml(v TomlConfig, prefix string, c *base.BaseConfig) bool {
changed := false
if k := prefix + "enabled"; v.IsSet(k) {
c.Enabled = v.GetBool(k)
changed = true
}
if k := prefix + "scan_interval_seconds"; v.IsSet(k) {
c.ScanIntervalSeconds = v.GetInt(k)
changed = true
}
if k := prefix + "max_concurrent"; v.IsSet(k) {
c.MaxConcurrent = v.GetInt(k)
changed = true
}
return changed
}
// ApplyPluginConfigFromToml overlays admin.toml [maintenance.*] settings onto
// existing plugin job type configs. Configs that do not exist yet get the
// overlay in applyPluginTomlDefaults when the plugin bootstraps them from a
// worker descriptor, so descriptor defaults are never lost.
func (s *AdminServer) ApplyPluginConfigFromToml(v TomlConfig) error {
plugin := s.GetPlugin()
if plugin == nil {
return nil
}
for _, section := range pluginConfigSections {
cfg, err := plugin.LoadJobTypeConfig(section.jobType)
if err != nil {
return fmt.Errorf("load %s plugin config: %w", section.jobType, err)
}
if cfg == nil {
continue
}
if !section.applyToml(v, cfg) {
continue
}
if err := plugin.SaveJobTypeConfig(cfg); err != nil {
return fmt.Errorf("save %s plugin config: %w", section.jobType, err)
}
glog.V(0).Infof("Applied [%s] settings from admin.toml to plugin config", section.prefix)
}
return nil
}
// applyPluginTomlDefaults overlays admin.toml onto a job type config the
// plugin is bootstrapping from descriptor defaults.
func applyPluginTomlDefaults(v TomlConfig, cfg *plugin_pb.PersistedJobTypeConfig) {
for _, section := range pluginConfigSections {
if section.jobType == cfg.JobType {
if section.applyToml(v, cfg) {
glog.V(0).Infof("Applied [%s] settings from admin.toml to plugin defaults", section.prefix)
}
return
}
}
}
type pluginConfigSection struct {
jobType string
prefix string
workerKeys map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue
adminKeys map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue
}
var pluginConfigSections = []pluginConfigSection{
{
jobType: "vacuum",
prefix: "maintenance.vacuum",
workerKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{
"garbage_threshold": doubleValue,
"min_volume_age_seconds": int64Value,
},
},
{
jobType: "volume_balance",
prefix: "maintenance.balance",
workerKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{
"imbalance_threshold": doubleValue,
"min_server_count": int64Value,
},
},
{
jobType: "erasure_coding",
prefix: "maintenance.erasure_coding",
workerKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{
"fullness_ratio": doubleValue,
"quiet_for_seconds": int64Value,
"min_size_mb": int64Value,
"preferred_tags": stringListValue,
"replica_placement": stringValue,
"strict_placement": boolValue,
"free_slots_deadband": int64Value,
"max_shard_transfers": int64Value,
},
// workers read collection_filter from the admin values, not the worker values
adminKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{
"collection_filter": stringValue,
},
},
{
// s3_lifecycle has no worker config form; its knobs are admin values
jobType: "s3_lifecycle",
prefix: "maintenance.s3_lifecycle",
adminKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{
"meta_log_retention_days": int64Value,
"walker_interval_minutes": int64Value,
"cluster_deletes_per_second": int64Value,
"cluster_deletes_burst": int64Value,
},
},
}
func doubleValue(v TomlConfig, key string) *plugin_pb.ConfigValue {
return &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: v.GetFloat64(key)}}
}
func int64Value(v TomlConfig, key string) *plugin_pb.ConfigValue {
return &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(v.GetInt(key))}}
}
func boolValue(v TomlConfig, key string) *plugin_pb.ConfigValue {
return &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_BoolValue{BoolValue: v.GetBool(key)}}
}
func stringValue(v TomlConfig, key string) *plugin_pb.ConfigValue {
return &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_StringValue{StringValue: v.GetString(key)}}
}
func stringListValue(v TomlConfig, key string) *plugin_pb.ConfigValue {
// viper does not split comma-separated values from env vars
var items []string
for _, item := range v.GetStringSlice(key) {
items = append(items, strings.Split(item, ",")...)
}
return &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_StringList{StringList: &plugin_pb.StringList{Values: util.NormalizeTagList(items)}}}
}
func (section pluginConfigSection) applyToml(v TomlConfig, cfg *plugin_pb.PersistedJobTypeConfig) bool {
changed := false
if k := section.prefix + ".enabled"; v.IsSet(k) {
ensureAdminRuntime(cfg).Enabled = v.GetBool(k)
changed = true
}
if k := section.prefix + ".retry_limit"; v.IsSet(k) {
ensureAdminRuntime(cfg).RetryLimit = int32Setting(v, k)
changed = true
}
if k := section.prefix + ".retry_backoff_seconds"; v.IsSet(k) {
ensureAdminRuntime(cfg).RetryBackoffSeconds = int32Setting(v, k)
changed = true
}
for tomlKey, mapper := range section.workerKeys {
if k := section.prefix + "." + tomlKey; v.IsSet(k) {
if cfg.WorkerConfigValues == nil {
cfg.WorkerConfigValues = make(map[string]*plugin_pb.ConfigValue)
}
cfg.WorkerConfigValues[tomlKey] = mapper(v, k)
changed = true
}
}
for tomlKey, mapper := range section.adminKeys {
if k := section.prefix + "." + tomlKey; v.IsSet(k) {
if cfg.AdminConfigValues == nil {
cfg.AdminConfigValues = make(map[string]*plugin_pb.ConfigValue)
}
cfg.AdminConfigValues[tomlKey] = mapper(v, k)
changed = true
}
}
if changed {
cfg.UpdatedAt = timestamppb.Now()
cfg.UpdatedBy = "admin.toml"
}
return changed
}
// int32Setting clamps to [0, MaxInt32] so oversized toml values cannot wrap negative
func int32Setting(v TomlConfig, key string) int32 {
n := v.GetInt(key)
if n < 0 {
return 0
}
if n > math.MaxInt32 {
return math.MaxInt32
}
return int32(n)
}
func ensureAdminRuntime(cfg *plugin_pb.PersistedJobTypeConfig) *plugin_pb.AdminRuntimeConfig {
if cfg.AdminRuntime == nil {
// maintenance job types are enabled by default; a bare runtime must not disable them
cfg.AdminRuntime = &plugin_pb.AdminRuntimeConfig{Enabled: true}
}
return cfg.AdminRuntime
}