mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-21 05:24:28 +02:00
fix(scheduler): give worker tasks a real per-attempt execution deadline (#9041)
* fix(scheduler): give worker tasks a real per-attempt execution deadline The plugin scheduler derived the per-attempt execution deadline as DetectionTimeoutSeconds * 2, which capped every worker task at twice the cluster-scan budget regardless of actual work. For volume_balance batches this was 240s — far too short for 20 large volume copies, so every attempt died at "context deadline exceeded" and all in-flight sub-RPCs surfaced as "context canceled". Retries restarted from move 1 and hit the same wall. Add an explicit ExecutionTimeoutSeconds field to the plugin proto and make each handler declare its own baseline (1800s for vacuum, balance, EC; 3600s for iceberg). Size-aware handlers also emit an estimated_runtime_seconds parameter on each proposal so the scheduler extends the per-attempt deadline based on actual workload: - volume_balance batch: max(largest single move, total / concurrency) at 5 min/GB, so a skewed batch with one big volume isn't averaged away. - volume_balance single, vacuum (already), erasure_coding (10 min/GB), ec_balance (5 min/GB): per-volume budgets. admin_script and iceberg keep the configurable handler default since their workloads are opaque to the detector. * fix(scheduler): apply descriptor defaults to existing persisted configs The previous commit added execution_timeout_seconds to the proto and each handler's descriptor defaults, but two paths still left existing deployments broken: 1. deriveSchedulerAdminRuntime returned stored AdminRuntime configs as-is. Persisted configs from older versions have no execution_timeout_seconds, so the scheduler fell back to the 90s default — worse than the prior 240s behavior. Overlay descriptor defaults for any zero numeric fields when loading. 2. The admin form did not round-trip execution_timeout_seconds, so a normal save would clear it back to zero. Add the input field, the fillAdminSettings/collectAdminSettings hooks, and as defense in depth reapply descriptor defaults in UpdatePluginJobTypeConfigAPI before persisting so a stale form can never silently clobber a baseline. * fix(volume_balance): account for partial scheduling rounds in batch estimate With N moves and C slots, the busiest slot processes ceil(N/C) moves, not N/C. Dividing total seconds by C underestimates wall-clock time whenever N is not a multiple of C — e.g. 6 moves at concurrency 5 needs 2 rounds, not 1.2. Use avg * ceil(N/C) so partial rounds are counted as full ones. * fix(volume_balance): scale minBudget per wave instead of per move Orchestration overhead (setup/teardown for the parallel move runner) happens once per wave, not once per move. Use numRounds*60 as the floor instead of len(moves)*60 so the minimum doesn't inflate linearly with batch size when individual moves are tiny.
This commit is contained in:
Chris Lu authored and GitHub committed
1 parent
28d1ef24ec
commit
ae08e77979
13 files changed
+158
-17
No files matched your search
@@ -2493,8 +2493,12 @@ type AdminRuntimeDefaults struct {
|
||||
RetryLimit int32 `protobuf:"varint,7,opt,name=retry_limit,json=retryLimit,proto3" json:"retry_limit,omitempty"`
|
||||
RetryBackoffSeconds int32 `protobuf:"varint,8,opt,name=retry_backoff_seconds,json=retryBackoffSeconds,proto3" json:"retry_backoff_seconds,omitempty"`
|
||||
JobTypeMaxRuntimeSeconds int32 `protobuf:"varint,9,opt,name=job_type_max_runtime_seconds,json=jobTypeMaxRuntimeSeconds,proto3" json:"job_type_max_runtime_seconds,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
// Per-attempt execution deadline for one job. Distinct from
|
||||
// detection_timeout (which bounds the detection scan) and
|
||||
// job_type_max_runtime (a lane-wide budget across all attempts).
|
||||
ExecutionTimeoutSeconds int32 `protobuf:"varint,10,opt,name=execution_timeout_seconds,json=executionTimeoutSeconds,proto3" json:"execution_timeout_seconds,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *AdminRuntimeDefaults) Reset() {
|
||||
@@ -2590,6 +2594,13 @@ func (x *AdminRuntimeDefaults) GetJobTypeMaxRuntimeSeconds() int32 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *AdminRuntimeDefaults) GetExecutionTimeoutSeconds() int32 {
|
||||
if x != nil {
|
||||
return x.ExecutionTimeoutSeconds
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type AdminRuntimeConfig struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"`
|
||||
@@ -2601,6 +2612,7 @@ type AdminRuntimeConfig struct {
|
||||
RetryLimit int32 `protobuf:"varint,7,opt,name=retry_limit,json=retryLimit,proto3" json:"retry_limit,omitempty"`
|
||||
RetryBackoffSeconds int32 `protobuf:"varint,8,opt,name=retry_backoff_seconds,json=retryBackoffSeconds,proto3" json:"retry_backoff_seconds,omitempty"`
|
||||
JobTypeMaxRuntimeSeconds int32 `protobuf:"varint,9,opt,name=job_type_max_runtime_seconds,json=jobTypeMaxRuntimeSeconds,proto3" json:"job_type_max_runtime_seconds,omitempty"`
|
||||
ExecutionTimeoutSeconds int32 `protobuf:"varint,10,opt,name=execution_timeout_seconds,json=executionTimeoutSeconds,proto3" json:"execution_timeout_seconds,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -2698,6 +2710,13 @@ func (x *AdminRuntimeConfig) GetJobTypeMaxRuntimeSeconds() int32 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *AdminRuntimeConfig) GetExecutionTimeoutSeconds() int32 {
|
||||
if x != nil {
|
||||
return x.ExecutionTimeoutSeconds
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type RunDetectionRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
|
||||
@@ -4091,7 +4110,7 @@ const file_plugin_proto_rawDesc = "" +
|
||||
"\x06fields\x18\x01 \x03(\v2\x1c.plugin.ValueMap.FieldsEntryR\x06fields\x1aN\n" +
|
||||
"\vFieldsEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
|
||||
"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\"\xff\x03\n" +
|
||||
"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\"\xbb\x04\n" +
|
||||
"\x14AdminRuntimeDefaults\x12\x18\n" +
|
||||
"\aenabled\x18\x01 \x01(\bR\aenabled\x12<\n" +
|
||||
"\x1adetection_interval_seconds\x18\x02 \x01(\x05R\x18detectionIntervalSeconds\x12:\n" +
|
||||
@@ -4102,7 +4121,9 @@ const file_plugin_proto_rawDesc = "" +
|
||||
"\vretry_limit\x18\a \x01(\x05R\n" +
|
||||
"retryLimit\x122\n" +
|
||||
"\x15retry_backoff_seconds\x18\b \x01(\x05R\x13retryBackoffSeconds\x12>\n" +
|
||||
"\x1cjob_type_max_runtime_seconds\x18\t \x01(\x05R\x18jobTypeMaxRuntimeSeconds\"\xfd\x03\n" +
|
||||
"\x1cjob_type_max_runtime_seconds\x18\t \x01(\x05R\x18jobTypeMaxRuntimeSeconds\x12:\n" +
|
||||
"\x19execution_timeout_seconds\x18\n" +
|
||||
" \x01(\x05R\x17executionTimeoutSeconds\"\xb9\x04\n" +
|
||||
"\x12AdminRuntimeConfig\x12\x18\n" +
|
||||
"\aenabled\x18\x01 \x01(\bR\aenabled\x12<\n" +
|
||||
"\x1adetection_interval_seconds\x18\x02 \x01(\x05R\x18detectionIntervalSeconds\x12:\n" +
|
||||
@@ -4113,7 +4134,9 @@ const file_plugin_proto_rawDesc = "" +
|
||||
"\vretry_limit\x18\a \x01(\x05R\n" +
|
||||
"retryLimit\x122\n" +
|
||||
"\x15retry_backoff_seconds\x18\b \x01(\x05R\x13retryBackoffSeconds\x12>\n" +
|
||||
"\x1cjob_type_max_runtime_seconds\x18\t \x01(\x05R\x18jobTypeMaxRuntimeSeconds\"\xef\x05\n" +
|
||||
"\x1cjob_type_max_runtime_seconds\x18\t \x01(\x05R\x18jobTypeMaxRuntimeSeconds\x12:\n" +
|
||||
"\x19execution_timeout_seconds\x18\n" +
|
||||
" \x01(\x05R\x17executionTimeoutSeconds\"\xef\x05\n" +
|
||||
"\x13RunDetectionRequest\x12\x1d\n" +
|
||||
"\n" +
|
||||
"request_id\x18\x01 \x01(\tR\trequestId\x12\x19\n" +
|
||||
|
||||
Reference in new issue
Block a user