Expire stuck plugin jobs (#8492)

* Add stale job expiry and expire API

* Add expire job button

* Add test hook and coverage for ExpirePluginJobAPI

* Document scheduler filtering side effect and reuse helper

* Restore job spec proposal test

* Regenerate plugin template output

---------

Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
Chris LuandCopilot authored and GitHub committed 2026-03-03 01:27:25 -08:00
1 parent 3db05f59f0
commit a61a2affe3
11 files changed
+548 -6

No files matched your search

+81 -4
View File
@@ -485,6 +485,11 @@ templ Plugin(page string) {
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="plugin-job-detail-modal-label"><i class="fas fa-file-alt me-2"></i>Job Detail</h5>
<div class="ms-auto me-2">
<button type="button" class="btn btn-outline-danger btn-sm" id="plugin-expire-job-btn" disabled>
<i class="fas fa-stop-circle me-1"></i>Expire Job
</button>
</div>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body" id="plugin-job-detail-content">
@@ -1073,6 +1078,66 @@ templ Plugin(page string) {
return html;
}
function isActiveJobState(candidateState) {
var jobState = candidateState;
if (candidateState && typeof candidateState === 'object' && candidateState.state !== undefined) {
jobState = candidateState.state;
}
var st = String(jobState || '').toLowerCase();
return st === 'job_state_pending' || st === 'job_state_assigned' || st === 'job_state_running' ||
st === 'pending' || st === 'assigned' || st === 'running' || st === 'in_progress';
}
function setExpireButtonState(job) {
var expireBtn = document.getElementById('plugin-expire-job-btn');
if (!expireBtn) {
return;
}
var jobID = job && job.job_id ? String(job.job_id) : '';
var active = isActiveJobState(job);
expireBtn.setAttribute('data-job-id', jobID);
expireBtn.disabled = !jobID || !active;
if (!jobID) {
expireBtn.title = 'Select a job to expire.';
} else if (!active) {
expireBtn.title = 'Job is not active.';
} else {
expireBtn.title = 'Expire job to unblock scheduling.';
}
}
async function expireJob(jobID) {
var normalizedJobID = String(jobID || '').trim();
if (!normalizedJobID) {
return;
}
var reason = window.prompt('Expire job ' + normalizedJobID + '? Optional reason:', 'job expired by admin request');
if (reason === null) {
return;
}
var expireBtn = document.getElementById('plugin-expire-job-btn');
if (expireBtn) {
expireBtn.disabled = true;
}
try {
var response = await pluginRequest('POST', '/api/plugin/jobs/' + encodePath(normalizedJobID) + '/expire', {
reason: reason,
});
if (response && response.expired === false) {
notify(response.message || 'Job is not active.', 'info');
} else {
notify('Job expired: ' + normalizedJobID, 'success');
}
await refreshJobsAndActivities();
await openJobDetail(normalizedJobID);
} catch (e) {
notify('Failed to expire job: ' + e.message, 'error');
}
}
async function openJobDetail(jobID) {
var normalizedJobID = String(jobID || '').trim();
if (!normalizedJobID) {
@@ -1093,10 +1158,12 @@ templ Plugin(page string) {
modal.show();
}
setExpireButtonState(null);
contentRoot.innerHTML = '<div class="text-muted">Loading job detail...</div>';
try {
var detail = await pluginRequest('GET', '/api/plugin/jobs/' + encodePath(normalizedJobID) + '/detail?activity_limit=500&related_limit=20');
var job = (detail && detail.job) ? detail.job : {};
setExpireButtonState(job);
var runRecord = detail && detail.run_record ? detail.run_record : null;
var activities = (detail && Array.isArray(detail.activities)) ? detail.activities : [];
var relatedJobs = (detail && Array.isArray(detail.related_jobs)) ? detail.related_jobs : [];
@@ -1197,6 +1264,7 @@ templ Plugin(page string) {
contentRoot.innerHTML = html;
} catch (e) {
setExpireButtonState(null);
contentRoot.innerHTML = '<div class="alert alert-danger mb-0">Failed to load job detail: ' + escapeHtml(e.message) + '</div>';
}
}
@@ -1238,8 +1306,7 @@ templ Plugin(page string) {
var allActivities = Array.isArray(state.allActivities) ? state.allActivities : [];
var activeCount = allJobs.filter(function(job) {
var st = String(job.state || '').toLowerCase();
return st === 'job_state_pending' || st === 'job_state_assigned' || st === 'job_state_running' || st === 'pending' || st === 'assigned' || st === 'running' || st === 'in_progress';
return isActiveJobState(job);
}).length;
document.getElementById('plugin-status-workers').textContent = String(state.workers.length);
@@ -1265,8 +1332,7 @@ templ Plugin(page string) {
if (!jobType) {
continue;
}
var st = String(job.state || '').toLowerCase();
var isActive = st === 'job_state_pending' || st === 'job_state_assigned' || st === 'job_state_running' || st === 'pending' || st === 'assigned' || st === 'running' || st === 'in_progress';
var isActive = isActiveJobState(job);
if (!isActive) {
continue;
}
@@ -2778,6 +2844,17 @@ templ Plugin(page string) {
});
}
var expireBtn = document.getElementById('plugin-expire-job-btn');
if (expireBtn) {
expireBtn.addEventListener('click', function() {
var jobID = String(expireBtn.getAttribute('data-job-id') || '').trim();
if (!jobID) {
return;
}
expireJob(jobID);
});
}
document.getElementById('plugin-refresh-all-btn').addEventListener('click', function() {
refreshAll();
});
+1 -1
View File
@@ -46,7 +46,7 @@ func Plugin(page string) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\"><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"d-flex justify-content-between align-items-center flex-wrap gap-2\"><div><h2 class=\"mb-0\"><i class=\"fas fa-plug me-2\"></i>Workers</h2><p class=\"text-muted mb-0\">Cluster-wide worker status, per-job configuration, detection, queue, and execution workflows.</p></div><div class=\"btn-group\"><button type=\"button\" class=\"btn btn-outline-secondary\" id=\"plugin-refresh-all-btn\"><i class=\"fas fa-sync-alt me-1\"></i>Refresh</button></div></div></div></div><div class=\"row mb-3\"><div class=\"col-12\"><ul class=\"nav nav-tabs\" id=\"plugin-top-tabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-top-tab=\"overview\"><i class=\"fas fa-chart-line me-1\"></i>Overview</button></li></ul></div></div><div class=\"row mb-3 d-none\" id=\"plugin-subtab-row\"><div class=\"col-12\"><ul class=\"nav nav-pills gap-2\" id=\"plugin-subtabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-subtab=\"configuration\"><i class=\"fas fa-sliders-h me-1\"></i>Configuration</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"detection\"><i class=\"fas fa-search me-1\"></i>Job Detection</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"queue\"><i class=\"fas fa-list me-1\"></i>Job Queue</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"execution\"><i class=\"fas fa-tasks me-1\"></i>Job Execution</button></li></ul></div></div><div class=\"plugin-section plugin-section-overview\"><div class=\"row mb-4\"><div class=\"col-md-4 mb-3\"><div class=\"card border-info h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Workers</div><h4 class=\"mb-0\" id=\"plugin-status-workers\">0</h4></div></div></div><div class=\"col-md-4 mb-3\"><div class=\"card border-warning h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Active Jobs</div><h4 class=\"mb-0\" id=\"plugin-status-active-jobs\">0</h4></div></div></div><div class=\"col-md-4 mb-3\"><div class=\"card border-success h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Activities (recent)</div><h4 class=\"mb-0\" id=\"plugin-status-activities\">0</h4></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"fas fa-chart-bar me-2\"></i>Per Job Type Summary</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Active Jobs</th><th>Recent Activities</th></tr></thead> <tbody id=\"plugin-jobtype-summary-body\"><tr><td colspan=\"3\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header d-flex justify-content-between align-items-center flex-wrap gap-2\"><h5 class=\"mb-0\"><i class=\"fas fa-clock me-2\"></i>Scheduler State</h5><small class=\"text-muted\">Per job type detection schedule and execution limits</small></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Enabled</th><th>Detector</th><th>In Flight</th><th>Next Detection</th><th>Interval</th><th>Exec Global</th><th>Exec/Worker</th><th>Executor Workers</th><th>Effective Exec</th></tr></thead> <tbody id=\"plugin-scheduler-table-body\"><tr><td colspan=\"10\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"fas fa-users me-2\"></i>Workers</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\" style=\"max-height: 560px; overflow-y: auto;\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light sticky-top\"><tr><th>Worker</th><th>Address</th><th>Capabilities</th><th>Load</th></tr></thead> <tbody id=\"plugin-workers-table-body\"><tr><td colspan=\"4\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div></div><div class=\"plugin-section plugin-section-configuration\"><div class=\"row mb-4\"><div class=\"col-lg-8 mb-3\"><div class=\"card shadow-sm h-100\"><div class=\"card-header d-flex justify-content-between align-items-center\"><h5 class=\"mb-0\"><i class=\"fas fa-sliders-h me-2\"></i>Job Type Configuration</h5><small class=\"text-muted\" id=\"plugin-config-updated-at\">Not loaded</small></div><div class=\"card-boLine truncated
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\"><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"d-flex justify-content-between align-items-center flex-wrap gap-2\"><div><h2 class=\"mb-0\"><i class=\"fas fa-plug me-2\"></i>Workers</h2><p class=\"text-muted mb-0\">Cluster-wide worker status, per-job configuration, detection, queue, and execution workflows.</p></div><div class=\"btn-group\"><button type=\"button\" class=\"btn btn-outline-secondary\" id=\"plugin-refresh-all-btn\"><i class=\"fas fa-sync-alt me-1\"></i>Refresh</button></div></div></div></div><div class=\"row mb-3\"><div class=\"col-12\"><ul class=\"nav nav-tabs\" id=\"plugin-top-tabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-top-tab=\"overview\"><i class=\"fas fa-chart-line me-1\"></i>Overview</button></li></ul></div></div><div class=\"row mb-3 d-none\" id=\"plugin-subtab-row\"><div class=\"col-12\"><ul class=\"nav nav-pills gap-2\" id=\"plugin-subtabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-subtab=\"configuration\"><i class=\"fas fa-sliders-h me-1\"></i>Configuration</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"detection\"><i class=\"fas fa-search me-1\"></i>Job Detection</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"queue\"><i class=\"fas fa-list me-1\"></i>Job Queue</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"execution\"><i class=\"fas fa-tasks me-1\"></i>Job Execution</button></li></ul></div></div><div class=\"plugin-section plugin-section-overview\"><div class=\"row mb-4\"><div class=\"col-md-4 mb-3\"><div class=\"card border-info h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Workers</div><h4 class=\"mb-0\" id=\"plugin-status-workers\">0</h4></div></div></div><div class=\"col-md-4 mb-3\"><div class=\"card border-warning h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Active Jobs</div><h4 class=\"mb-0\" id=\"plugin-status-active-jobs\">0</h4></div></div></div><div class=\"col-md-4 mb-3\"><div class=\"card border-success h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Activities (recent)</div><h4 class=\"mb-0\" id=\"plugin-status-activities\">0</h4></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"fas fa-chart-bar me-2\"></i>Per Job Type Summary</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Active Jobs</th><th>Recent Activities</th></tr></thead> <tbody id=\"plugin-jobtype-summary-body\"><tr><td colspan=\"3\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header d-flex justify-content-between align-items-center flex-wrap gap-2\"><h5 class=\"mb-0\"><i class=\"fas fa-clock me-2\"></i>Scheduler State</h5><small class=\"text-muted\">Per job type detection schedule and execution limits</small></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Enabled</th><th>Detector</th><th>In Flight</th><th>Next Detection</th><th>Interval</th><th>Exec Global</th><th>Exec/Worker</th><th>Executor Workers</th><th>Effective Exec</th></tr></thead> <tbody id=\"plugin-scheduler-table-body\"><tr><td colspan=\"10\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"fas fa-users me-2\"></i>Workers</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\" style=\"max-height: 560px; overflow-y: auto;\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light sticky-top\"><tr><th>Worker</th><th>Address</th><th>Capabilities</th><th>Load</th></tr></thead> <tbody id=\"plugin-workers-table-body\"><tr><td colspan=\"4\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div></div><div class=\"plugin-section plugin-section-configuration\"><div class=\"row mb-4\"><div class=\"col-lg-8 mb-3\"><div class=\"card shadow-sm h-100\"><div class=\"card-header d-flex justify-content-between align-items-center\"><h5 class=\"mb-0\"><i class=\"fas fa-sliders-h me-2\"></i>Job Type Configuration</h5><small class=\"text-muted\" id=\"plugin-config-updated-at\">Not loaded</small></div><div class=\"card-boLine truncated
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}