s3 lifecycle: report work-based job progress during a pass (#12644)

* s3 lifecycle: report work-based job progress during a pass

The daily pass sent progress only while starting, so a healthy scan that
ran for many hours could reach the admin's 24-hour no-progress guard.
Worker heartbeats prove the worker is alive, not that this job moves.

While dailyrun.Run is in progress the handler now sends a progress update
every 5 minutes, and only when the pass's counters (entries visited,
replay events, RPC outcomes including errors) moved since the last
delivered update. An idle tick sends nothing, so a stalled pass still
expires. A failed send is retried on the next tick. The reporter is
stopped and joined before the completion is sent, so no progress can
follow the terminal result.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* admin: test that progress, not heartbeats, keeps an old job active

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3 lifecycle: label progress phase and bucket as the last started

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

weed/admin/plugin/plugin_monitor_test.go

@@ -793,6 +793,64 @@ func TestExpireStaleJobsCancelsTheJobOnItsWorker(t *testing.T) {
 	expectCancelFor(t, session, "req-stale")
 }
 
+// A job older than the stale limit stays active while its worker reports
+// progress for it; the worker's heartbeat alone does not keep it alive.
+func TestExpireStaleJobsKeepsAJobThatReportsProgress(t *testing.T) {
+	t.Parallel()
+
+	pluginSvc, err := New(Options{})
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	defer pluginSvc.Shutdown()
+
+	const workerID = "worker-1"
+	session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
+	pluginSvc.putSession(session)
+	pluginSvc.handleWorkerMessage(session, &plugin_pb.WorkerToAdminMessage{
+		Body: &plugin_pb.WorkerToAdminMessage_Hello{Hello: &plugin_pb.WorkerHello{WorkerId: workerID}},
+	})
+	pluginSvc.trackExecutionStart("req-working", workerID, &plugin_pb.JobSpec{JobId: "job-working", JobType: "s3_lifecycle"}, 1)
+	pluginSvc.trackExecutionStart("req-stalled", workerID, &plugin_pb.JobSpec{JobId: "job-stalled", JobType: "s3_lifecycle"}, 1)
+
+	startedAt := time.Now().UTC().Add(-defaultStaleActiveJobTimeout - time.Hour)
+	pluginSvc.jobsMu.Lock()
+	for _, job := range pluginSvc.jobs {
+		job.CreatedAt = timeToPtr(startedAt)
+		job.UpdatedAt = timeToPtr(startedAt)
+	}
+	pluginSvc.jobsMu.Unlock()
+
+	pluginSvc.handleWorkerMessage(session, &plugin_pb.WorkerToAdminMessage{
+		Body: &plugin_pb.WorkerToAdminMessage_Heartbeat{Heartbeat: &plugin_pb.WorkerHeartbeat{
+			WorkerId:            workerID,
+			ExecutionSlotsTotal: 4,
+			ExecutionSlotsUsed:  2,
+			RunningWork: []*plugin_pb.RunningWork{
+				{WorkId: "job-working", Kind: plugin_pb.WorkKind_WORK_KIND_EXECUTION, JobType: "s3_lifecycle"},
+				{WorkId: "job-stalled", Kind: plugin_pb.WorkKind_WORK_KIND_EXECUTION, JobType: "s3_lifecycle"},
+			},
+		}},
+	})
+	pluginSvc.handleWorkerMessage(session, &plugin_pb.WorkerToAdminMessage{
+		Body: &plugin_pb.WorkerToAdminMessage_JobProgressUpdate{JobProgressUpdate: &plugin_pb.JobProgressUpdate{
+			JobId: "job-working", JobType: "s3_lifecycle", RequestId: "req-working",
+			State: plugin_pb.JobState_JOB_STATE_RUNNING, Stage: "walk", Message: "entries=10",
+		}},
+	})
+
+	if n := pluginSvc.expireStaleJobs(time.Now().UTC()); n != 1 {
+		t.Fatalf("expired %d jobs, want only the stalled one", n)
+	}
+	if got, _ := pluginSvc.GetTrackedJob("job-working"); got == nil || got.State != "job_state_running" {
+		t.Fatalf("expected the job reporting progress to stay running, got %+v", got)
+	}
+	if got, _ := pluginSvc.GetTrackedJob("job-stalled"); got == nil || got.State != StateFailed {
+		t.Fatalf("expected the heartbeat-only job to be failed, got %+v", got)
+	}
+	expectCancelFor(t, session, "req-stalled")
+}
+
 func expectNoCancel(t *testing.T, session *streamSession) {
 	t.Helper()
 	select {