add admin script worker (#8491)

* admin: add plugin lock coordination

* shell: allow bypassing lock checks

* plugin worker: add admin script handler

* mini: include admin_script in plugin defaults

* admin script UI: drop name and enlarge text

* admin script: add default script

* admin_script: make run interval configurable

* plugin: gate other jobs during admin_script runs

* plugin: use last completed admin_script run

* admin: backfill plugin config defaults

* templ

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* comparable to default version

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* default to run

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* format

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* shell: respect pre-set noLock for fix.replication

* shell: add force no-lock mode for admin scripts

* volume balance worker already exists

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* admin: expose scheduler status JSON

* shell: add sleep command

* shell: restrict sleep syntax

* Revert "shell: respect pre-set noLock for fix.replication"

This reverts commit 2b14e8b826.

* templ

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* fix import

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* less logs

Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>

* Reduce master client logs on canceled contexts

* Update mini default job type count

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

weed/admin/plugin/plugin_scheduler.go

@@ -321,6 +321,22 @@ func (r *Plugin) pruneDetectorLeases(activeJobTypes map[string]struct{}) {
 func (r *Plugin) runScheduledDetection(jobType string, policy schedulerPolicy) {
 	defer r.finishDetection(jobType)
 
+	releaseLock, lockErr := r.acquireAdminLock(fmt.Sprintf("plugin scheduled detection %s", jobType))
+	if lockErr != nil {
+		r.recordSchedulerDetectionError(jobType, lockErr)
+		r.appendActivity(JobActivity{
+			JobType:    jobType,
+			Source:     "admin_scheduler",
+			Message:    fmt.Sprintf("scheduled detection aborted: failed to acquire lock: %v", lockErr),
+			Stage:      "failed",
+			OccurredAt: timeToPtr(time.Now().UTC()),
+		})
+		return
+	}
+	if releaseLock != nil {
+		defer releaseLock()
+	}
+
 	start := time.Now().UTC()
 	r.appendActivity(JobActivity{
 		JobType:    jobType,
@@ -331,6 +347,7 @@ func (r *Plugin) runScheduledDetection(jobType string, policy schedulerPolicy) {
 	})
 
 	if skip, waitingCount, waitingThreshold := r.shouldSkipDetectionForWaitingJobs(jobType, policy); skip {
+		r.recordSchedulerDetectionSkip(jobType, fmt.Sprintf("waiting backlog %d reached threshold %d", waitingCount, waitingThreshold))
 		r.appendActivity(JobActivity{
 			JobType:    jobType,
 			Source:     "admin_scheduler",
@@ -343,6 +360,7 @@ func (r *Plugin) runScheduledDetection(jobType string, policy schedulerPolicy) {
 
 	clusterContext, err := r.loadSchedulerClusterContext()
 	if err != nil {
+		r.recordSchedulerDetectionError(jobType, err)
 		r.appendActivity(JobActivity{
 			JobType:    jobType,
 			Source:     "admin_scheduler",
@@ -357,6 +375,7 @@ func (r *Plugin) runScheduledDetection(jobType string, policy schedulerPolicy) {
 	proposals, err := r.RunDetection(ctx, jobType, clusterContext, policy.MaxResults)
 	cancel()
 	if err != nil {
+		r.recordSchedulerDetectionError(jobType, err)
 		r.appendActivity(JobActivity{
 			JobType:    jobType,
 			Source:     "admin_scheduler",
@@ -374,6 +393,7 @@ func (r *Plugin) runScheduledDetection(jobType string, policy schedulerPolicy) {
 		Stage:      "detected",
 		OccurredAt: timeToPtr(time.Now().UTC()),
 	})
+	r.recordSchedulerDetectionSuccess(jobType, len(proposals))
 
 	filteredByActive, skippedActive := r.filterProposalsWithActiveJobs(jobType, proposals)
 	if skippedActive > 0 {