add admin script worker (#8491)

* admin: add plugin lock coordination

* shell: allow bypassing lock checks

* plugin worker: add admin script handler

* mini: include admin_script in plugin defaults

* admin script UI: drop name and enlarge text

* admin script: add default script

* admin_script: make run interval configurable

* plugin: gate other jobs during admin_script runs

* plugin: use last completed admin_script run

* admin: backfill plugin config defaults

* templ

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* comparable to default version

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* default to run

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* format

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* shell: respect pre-set noLock for fix.replication

* shell: add force no-lock mode for admin scripts

* volume balance worker already exists

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* admin: expose scheduler status JSON

* shell: add sleep command

* shell: restrict sleep syntax

* Revert "shell: respect pre-set noLock for fix.replication"

This reverts commit 2b14e8b826.

* templ

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* fix import

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* less logs

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* Reduce master client logs on canceled contexts

* Update mini default job type count

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

weed/admin/dash/plugin_api_test.go

@@ -140,3 +140,83 @@ func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
 		t.Fatalf("dedupe key must be preserved: got=%s want=%s", jobA.DedupeKey, proposal.DedupeKey)
 	}
 }
+
+func TestApplyDescriptorDefaultsToPersistedConfigBackfillsAdminDefaults(t *testing.T) {
+	t.Parallel()
+
+	config := &plugin_pb.PersistedJobTypeConfig{
+		JobType:            "admin_script",
+		AdminConfigValues:  map[string]*plugin_pb.ConfigValue{},
+		WorkerConfigValues: map[string]*plugin_pb.ConfigValue{},
+		AdminRuntime:       &plugin_pb.AdminRuntimeConfig{},
+	}
+	descriptor := &plugin_pb.JobTypeDescriptor{
+		JobType: "admin_script",
+		AdminConfigForm: &plugin_pb.ConfigForm{
+			DefaultValues: map[string]*plugin_pb.ConfigValue{
+				"script": {
+					Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "volume.balance -apply"},
+				},
+				"run_interval_minutes": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 17},
+				},
+			},
+		},
+		AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
+			DetectionIntervalSeconds: 60,
+			DetectionTimeoutSeconds:  300,
+		},
+	}
+
+	applyDescriptorDefaultsToPersistedConfig(config, descriptor)
+
+	script := config.AdminConfigValues["script"]
+	if script == nil {
+		t.Fatalf("expected script default to be backfilled")
+	}
+	scriptKind, ok := script.Kind.(*plugin_pb.ConfigValue_StringValue)
+	if !ok || scriptKind.StringValue == "" {
+		t.Fatalf("expected non-empty script default, got=%+v", script)
+	}
+	if config.AdminRuntime.DetectionIntervalSeconds != 60 {
+		t.Fatalf("expected runtime detection interval default to be backfilled")
+	}
+}
+
+func TestApplyDescriptorDefaultsToPersistedConfigReplacesBlankAdminScript(t *testing.T) {
+	t.Parallel()
+
+	config := &plugin_pb.PersistedJobTypeConfig{
+		JobType: "admin_script",
+		AdminConfigValues: map[string]*plugin_pb.ConfigValue{
+			"script": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "   "},
+			},
+		},
+		AdminRuntime: &plugin_pb.AdminRuntimeConfig{},
+	}
+	descriptor := &plugin_pb.JobTypeDescriptor{
+		JobType: "admin_script",
+		AdminConfigForm: &plugin_pb.ConfigForm{
+			DefaultValues: map[string]*plugin_pb.ConfigValue{
+				"script": {
+					Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "volume.fix.replication -apply"},
+				},
+			},
+		},
+	}
+
+	applyDescriptorDefaultsToPersistedConfig(config, descriptor)
+
+	script := config.AdminConfigValues["script"]
+	if script == nil {
+		t.Fatalf("expected script config value")
+	}
+	scriptKind, ok := script.Kind.(*plugin_pb.ConfigValue_StringValue)
+	if !ok {
+		t.Fatalf("expected string script config value, got=%T", script.Kind)
+	}
+	if scriptKind.StringValue != "volume.fix.replication -apply" {
+		t.Fatalf("expected blank script to be replaced by default, got=%q", scriptKind.StringValue)
+	}
+}