mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 08:45:22 +01:00
Expire stuck plugin jobs (#8492)
* Add stale job expiry and expire API * Add expire job button * Add test hook and coverage for ExpirePluginJobAPI * Document scheduler filtering side effect and reuse helper * Restore job spec proposal test * Regenerate plugin template output --------- Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
1 parent
3db05f59f0
commit
a61a2affe3
11 files changed
+548
-6
No files matched your search
@@ -485,6 +485,11 @@ templ Plugin(page string) {
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="plugin-job-detail-modal-label"><i class="fas fa-file-alt me-2"></i>Job Detail</h5>
|
||||
<div class="ms-auto me-2">
|
||||
<button type="button" class="btn btn-outline-danger btn-sm" id="plugin-expire-job-btn" disabled>
|
||||
<i class="fas fa-stop-circle me-1"></i>Expire Job
|
||||
</button>
|
||||
</div>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body" id="plugin-job-detail-content">
|
||||
@@ -1073,6 +1078,66 @@ templ Plugin(page string) {
|
||||
return html;
|
||||
}
|
||||
|
||||
function isActiveJobState(candidateState) {
|
||||
var jobState = candidateState;
|
||||
if (candidateState && typeof candidateState === 'object' && candidateState.state !== undefined) {
|
||||
jobState = candidateState.state;
|
||||
}
|
||||
var st = String(jobState || '').toLowerCase();
|
||||
return st === 'job_state_pending' || st === 'job_state_assigned' || st === 'job_state_running' ||
|
||||
st === 'pending' || st === 'assigned' || st === 'running' || st === 'in_progress';
|
||||
}
|
||||
|
||||
function setExpireButtonState(job) {
|
||||
var expireBtn = document.getElementById('plugin-expire-job-btn');
|
||||
if (!expireBtn) {
|
||||
return;
|
||||
}
|
||||
var jobID = job && job.job_id ? String(job.job_id) : '';
|
||||
var active = isActiveJobState(job);
|
||||
expireBtn.setAttribute('data-job-id', jobID);
|
||||
expireBtn.disabled = !jobID || !active;
|
||||
if (!jobID) {
|
||||
expireBtn.title = 'Select a job to expire.';
|
||||
} else if (!active) {
|
||||
expireBtn.title = 'Job is not active.';
|
||||
} else {
|
||||
expireBtn.title = 'Expire job to unblock scheduling.';
|
||||
}
|
||||
}
|
||||
|
||||
async function expireJob(jobID) {
|
||||
var normalizedJobID = String(jobID || '').trim();
|
||||
if (!normalizedJobID) {
|
||||
return;
|
||||
}
|
||||
|
||||
var reason = window.prompt('Expire job ' + normalizedJobID + '? Optional reason:', 'job expired by admin request');
|
||||
if (reason === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
var expireBtn = document.getElementById('plugin-expire-job-btn');
|
||||
if (expireBtn) {
|
||||
expireBtn.disabled = true;
|
||||
}
|
||||
|
||||
try {
|
||||
var response = await pluginRequest('POST', '/api/plugin/jobs/' + encodePath(normalizedJobID) + '/expire', {
|
||||
reason: reason,
|
||||
});
|
||||
if (response && response.expired === false) {
|
||||
notify(response.message || 'Job is not active.', 'info');
|
||||
} else {
|
||||
notify('Job expired: ' + normalizedJobID, 'success');
|
||||
}
|
||||
await refreshJobsAndActivities();
|
||||
await openJobDetail(normalizedJobID);
|
||||
} catch (e) {
|
||||
notify('Failed to expire job: ' + e.message, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
async function openJobDetail(jobID) {
|
||||
var normalizedJobID = String(jobID || '').trim();
|
||||
if (!normalizedJobID) {
|
||||
@@ -1093,10 +1158,12 @@ templ Plugin(page string) {
|
||||
modal.show();
|
||||
}
|
||||
|
||||
setExpireButtonState(null);
|
||||
contentRoot.innerHTML = '<div class="text-muted">Loading job detail...</div>';
|
||||
try {
|
||||
var detail = await pluginRequest('GET', '/api/plugin/jobs/' + encodePath(normalizedJobID) + '/detail?activity_limit=500&related_limit=20');
|
||||
var job = (detail && detail.job) ? detail.job : {};
|
||||
setExpireButtonState(job);
|
||||
var runRecord = detail && detail.run_record ? detail.run_record : null;
|
||||
var activities = (detail && Array.isArray(detail.activities)) ? detail.activities : [];
|
||||
var relatedJobs = (detail && Array.isArray(detail.related_jobs)) ? detail.related_jobs : [];
|
||||
@@ -1197,6 +1264,7 @@ templ Plugin(page string) {
|
||||
|
||||
contentRoot.innerHTML = html;
|
||||
} catch (e) {
|
||||
setExpireButtonState(null);
|
||||
contentRoot.innerHTML = '<div class="alert alert-danger mb-0">Failed to load job detail: ' + escapeHtml(e.message) + '</div>';
|
||||
}
|
||||
}
|
||||
@@ -1238,8 +1306,7 @@ templ Plugin(page string) {
|
||||
var allActivities = Array.isArray(state.allActivities) ? state.allActivities : [];
|
||||
|
||||
var activeCount = allJobs.filter(function(job) {
|
||||
var st = String(job.state || '').toLowerCase();
|
||||
return st === 'job_state_pending' || st === 'job_state_assigned' || st === 'job_state_running' || st === 'pending' || st === 'assigned' || st === 'running' || st === 'in_progress';
|
||||
return isActiveJobState(job);
|
||||
}).length;
|
||||
|
||||
document.getElementById('plugin-status-workers').textContent = String(state.workers.length);
|
||||
@@ -1265,8 +1332,7 @@ templ Plugin(page string) {
|
||||
if (!jobType) {
|
||||
continue;
|
||||
}
|
||||
var st = String(job.state || '').toLowerCase();
|
||||
var isActive = st === 'job_state_pending' || st === 'job_state_assigned' || st === 'job_state_running' || st === 'pending' || st === 'assigned' || st === 'running' || st === 'in_progress';
|
||||
var isActive = isActiveJobState(job);
|
||||
if (!isActive) {
|
||||
continue;
|
||||
}
|
||||
@@ -2778,6 +2844,17 @@ templ Plugin(page string) {
|
||||
});
|
||||
}
|
||||
|
||||
var expireBtn = document.getElementById('plugin-expire-job-btn');
|
||||
if (expireBtn) {
|
||||
expireBtn.addEventListener('click', function() {
|
||||
var jobID = String(expireBtn.getAttribute('data-job-id') || '').trim();
|
||||
if (!jobID) {
|
||||
return;
|
||||
}
|
||||
expireJob(jobID);
|
||||
});
|
||||
}
|
||||
|
||||
document.getElementById('plugin-refresh-all-btn').addEventListener('click', function() {
|
||||
refreshAll();
|
||||
});
|
||||
|
||||
@@ -46,7 +46,7 @@ func Plugin(page string) templ.Component {
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\"><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"d-flex justify-content-between align-items-center flex-wrap gap-2\"><div><h2 class=\"mb-0\"><i class=\"fas fa-plug me-2\"></i>Workers</h2><p class=\"text-muted mb-0\">Cluster-wide worker status, per-job configuration, detection, queue, and execution workflows.</p></div><div class=\"btn-group\"><button type=\"button\" class=\"btn btn-outline-secondary\" id=\"plugin-refresh-all-btn\"><i class=\"fas fa-sync-alt me-1\"></i>Refresh</button></div></div></div></div><div class=\"row mb-3\"><div class=\"col-12\"><ul class=\"nav nav-tabs\" id=\"plugin-top-tabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-top-tab=\"overview\"><i class=\"fas fa-chart-line me-1\"></i>Overview</button></li></ul></div></div><div class=\"row mb-3 d-none\" id=\"plugin-subtab-row\"><div class=\"col-12\"><ul class=\"nav nav-pills gap-2\" id=\"plugin-subtabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-subtab=\"configuration\"><i class=\"fas fa-sliders-h me-1\"></i>Configuration</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"detection\"><i class=\"fas fa-search me-1\"></i>Job Detection</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"queue\"><i class=\"fas fa-list me-1\"></i>Job Queue</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"execution\"><i class=\"fas fa-tasks me-1\"></i>Job Execution</button></li></ul></div></div><div class=\"plugin-section plugin-section-overview\"><div class=\"row mb-4\"><div class=\"col-md-4 mb-3\"><div class=\"card border-info h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Workers</div><h4 class=\"mb-0\" id=\"plugin-status-workers\">0</h4></div></div></div><div class=\"col-md-4 mb-3\"><div class=\"card border-warning h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Active Jobs</div><h4 class=\"mb-0\" id=\"plugin-status-active-jobs\">0</h4></div></div></div><div class=\"col-md-4 mb-3\"><div class=\"card border-success h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Activities (recent)</div><h4 class=\"mb-0\" id=\"plugin-status-activities\">0</h4></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"fas fa-chart-bar me-2\"></i>Per Job Type Summary</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Active Jobs</th><th>Recent Activities</th></tr></thead> <tbody id=\"plugin-jobtype-summary-body\"><tr><td colspan=\"3\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header d-flex justify-content-between align-items-center flex-wrap gap-2\"><h5 class=\"mb-0\"><i class=\"fas fa-clock me-2\"></i>Scheduler State</h5><small class=\"text-muted\">Per job type detection schedule and execution limits</small></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Enabled</th><th>Detector</th><th>In Flight</th><th>Next Detection</th><th>Interval</th><th>Exec Global</th><th>Exec/Worker</th><th>Executor Workers</th><th>Effective Exec</th></tr></thead> <tbody id=\"plugin-scheduler-table-body\"><tr><td colspan=\"10\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"fas fa-users me-2\"></i>Workers</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\" style=\"max-height: 560px; overflow-y: auto;\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light sticky-top\"><tr><th>Worker</th><th>Address</th><th>Capabilities</th><th>Load</th></tr></thead> <tbody id=\"plugin-workers-table-body\"><tr><td colspan=\"4\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div></div><div class=\"plugin-section plugin-section-configuration\"><div class=\"row mb-4\"><div class=\"col-lg-8 mb-3\"><div class=\"card shadow-sm h-100\"><div class=\"card-header d-flex justify-content-between align-items-center\"><h5 class=\"mb-0\"><i class=\"fas fa-sliders-h me-2\"></i>Job Type Configuration</h5><small class=\"text-muted\" id=\"plugin-config-updated-at\">Not loaded</small></div><div class=\"card-boLine truncated
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\"><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"d-flex justify-content-between align-items-center flex-wrap gap-2\"><div><h2 class=\"mb-0\"><i class=\"fas fa-plug me-2\"></i>Workers</h2><p class=\"text-muted mb-0\">Cluster-wide worker status, per-job configuration, detection, queue, and execution workflows.</p></div><div class=\"btn-group\"><button type=\"button\" class=\"btn btn-outline-secondary\" id=\"plugin-refresh-all-btn\"><i class=\"fas fa-sync-alt me-1\"></i>Refresh</button></div></div></div></div><div class=\"row mb-3\"><div class=\"col-12\"><ul class=\"nav nav-tabs\" id=\"plugin-top-tabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-top-tab=\"overview\"><i class=\"fas fa-chart-line me-1\"></i>Overview</button></li></ul></div></div><div class=\"row mb-3 d-none\" id=\"plugin-subtab-row\"><div class=\"col-12\"><ul class=\"nav nav-pills gap-2\" id=\"plugin-subtabs\"><li class=\"nav-item\"><button type=\"button\" class=\"nav-link active\" data-plugin-subtab=\"configuration\"><i class=\"fas fa-sliders-h me-1\"></i>Configuration</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"detection\"><i class=\"fas fa-search me-1\"></i>Job Detection</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"queue\"><i class=\"fas fa-list me-1\"></i>Job Queue</button></li><li class=\"nav-item\"><button type=\"button\" class=\"nav-link\" data-plugin-subtab=\"execution\"><i class=\"fas fa-tasks me-1\"></i>Job Execution</button></li></ul></div></div><div class=\"plugin-section plugin-section-overview\"><div class=\"row mb-4\"><div class=\"col-md-4 mb-3\"><div class=\"card border-info h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Workers</div><h4 class=\"mb-0\" id=\"plugin-status-workers\">0</h4></div></div></div><div class=\"col-md-4 mb-3\"><div class=\"card border-warning h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Active Jobs</div><h4 class=\"mb-0\" id=\"plugin-status-active-jobs\">0</h4></div></div></div><div class=\"col-md-4 mb-3\"><div class=\"card border-success h-100\"><div class=\"card-body text-center\"><div class=\"text-uppercase text-muted small mb-1\">Activities (recent)</div><h4 class=\"mb-0\" id=\"plugin-status-activities\">0</h4></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"fas fa-chart-bar me-2\"></i>Per Job Type Summary</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Active Jobs</th><th>Recent Activities</th></tr></thead> <tbody id=\"plugin-jobtype-summary-body\"><tr><td colspan=\"3\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header d-flex justify-content-between align-items-center flex-wrap gap-2\"><h5 class=\"mb-0\"><i class=\"fas fa-clock me-2\"></i>Scheduler State</h5><small class=\"text-muted\">Per job type detection schedule and execution limits</small></div><div class=\"card-body p-0\"><div class=\"table-responsive\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light\"><tr><th>Job Type</th><th>Enabled</th><th>Detector</th><th>In Flight</th><th>Next Detection</th><th>Interval</th><th>Exec Global</th><th>Exec/Worker</th><th>Executor Workers</th><th>Effective Exec</th></tr></thead> <tbody id=\"plugin-scheduler-table-body\"><tr><td colspan=\"10\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div><div class=\"row mb-4\"><div class=\"col-12\"><div class=\"card shadow-sm\"><div class=\"card-header\"><h5 class=\"mb-0\"><i class=\"fas fa-users me-2\"></i>Workers</h5></div><div class=\"card-body p-0\"><div class=\"table-responsive\" style=\"max-height: 560px; overflow-y: auto;\"><table class=\"table table-sm table-hover mb-0\"><thead class=\"table-light sticky-top\"><tr><th>Worker</th><th>Address</th><th>Capabilities</th><th>Load</th></tr></thead> <tbody id=\"plugin-workers-table-body\"><tr><td colspan=\"4\" class=\"text-muted text-center py-3\">Loading...</td></tr></tbody></table></div></div></div></div></div></div><div class=\"plugin-section plugin-section-configuration\"><div class=\"row mb-4\"><div class=\"col-lg-8 mb-3\"><div class=\"card shadow-sm h-100\"><div class=\"card-header d-flex justify-content-between align-items-center\"><h5 class=\"mb-0\"><i class=\"fas fa-sliders-h me-2\"></i>Job Type Configuration</h5><small class=\"text-muted\" id=\"plugin-config-updated-at\">Not loaded</small></div><div class=\"card-boLine truncated
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user