seaweedfs/weed/admin/plugin/plugin_detection_test.go

package plugin

import (
	"context"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
	"github.com/stretchr/testify/require"
)

func TestScheduledDetectionDistinguishesSkipFromEmptyScan(t *testing.T) {
	for _, reason := range []string{"", "Cooldown after successful job (10h0m0s)"} {
		t.Run(reason, func(t *testing.T) {
			p, err := New(Options{})
			require.NoError(t, err)
			defer p.Shutdown()
			p.clusterContextProvider = func(context.Context) (*plugin_pb.ClusterContext, error) {
				return &plugin_pb.ClusterContext{}, nil
			}
			p.registry.UpsertFromHello(&plugin_pb.WorkerHello{
				WorkerId: "worker-a", Capabilities: []*plugin_pb.JobTypeCapability{{JobType: "vacuum", CanDetect: true}},
			})
			session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
			p.putSession(session)
			done := make(chan struct{})
			go func() {
				p.runJobTypeIteration("vacuum", schedulerPolicy{DetectionTimeout: time.Second, JobTypeMaxRuntime: time.Second})
				close(done)
			}()
			select {
			case request := <-session.outgoing:
				p.handleDetectionComplete("worker-a", &plugin_pb.DetectionComplete{
					RequestId: request.RequestId, JobType: "vacuum", Success: true, SkippedReason: reason,
				})
			case <-time.After(2 * time.Second):
				t.Fatal("no detection request")
			}
			select {
			case <-done:
			case <-time.After(2 * time.Second):
				t.Fatal("detection did not complete")
			}
			info := p.snapshotSchedulerDetection("vacuum")
			if reason == "" {
				require.False(t, info.lastDetectedAt.IsZero())
				require.Equal(t, "success", p.snapshotSchedulerRun("vacuum").lastRunStatus)
			} else {
				require.True(t, info.lastDetectedAt.IsZero(), "a skipped scan must not count as an empty scan")
				require.Equal(t, reason, info.lastSkippedReason)
				require.Equal(t, "skipped", p.snapshotSchedulerRun("vacuum").lastRunStatus)
			}
		})
	}
}

func TestManualDetectionSkipDoesNotChangeScheduledStatus(t *testing.T) {
	p, err := New(Options{})
	require.NoError(t, err)
	defer p.Shutdown()
	p.recordSchedulerDetectionSuccess("vacuum", 5)
	p.pendingDetection["manual"] = &pendingDetectionState{
		jobType: "vacuum", workerID: "worker-a", complete: make(chan *plugin_pb.DetectionComplete, 1),
	}
	p.handleDetectionComplete("worker-a", &plugin_pb.DetectionComplete{
		RequestId: "manual", JobType: "vacuum", Success: true, SkippedReason: "Cooldown",
	})
	require.Empty(t, p.snapshotSchedulerDetection("vacuum").lastSkippedReason)
	require.Equal(t, 5, p.snapshotSchedulerDetection("vacuum").lastDetectedCount)
	activities := p.ListActivities("vacuum", 10)
	require.Len(t, activities, 1)
	require.Equal(t, "skipped", activities[0].Stage)
}

func TestPrefetchRefreshesSavedDescriptorFromReconnectingWorker(t *testing.T) {
	p, err := New(Options{})
	require.NoError(t, err)
	defer p.Shutdown()
	require.NoError(t, p.store.SaveDescriptor("vacuum", &plugin_pb.JobTypeDescriptor{JobType: "vacuum", DescriptorVersion: 1}))
	require.NoError(t, p.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
		JobType: "vacuum", WorkerConfigValues: map[string]*plugin_pb.ConfigValue{"cooldown": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 600}}},
	}))
	session := &streamSession{workerID: "upgraded", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
	p.putSession(session)
	done := make(chan struct{})
	go func() {
		p.prefetchDescriptorsFromHello(&plugin_pb.WorkerHello{
			WorkerId: "upgraded", Capabilities: []*plugin_pb.JobTypeCapability{{JobType: "vacuum", CanDetect: true}},
		})
		close(done)
	}()
	select {
	case request := <-session.outgoing:
		require.True(t, request.GetRequestConfigSchema().GetForceRefresh())
		p.handleConfigSchemaResponse(&plugin_pb.ConfigSchemaResponse{
			RequestId: request.RequestId, JobType: "vacuum", Success: true,
			JobTypeDescriptor: &plugin_pb.JobTypeDescriptor{JobType: "vacuum", DescriptorVersion: 2, DetectionCooldownConfigField: "cooldown"},
		})
	case <-time.After(time.Second):
		t.Fatal("saved descriptor was not refreshed")
	}
	select {
	case <-done:
	case <-time.After(time.Second):
		t.Fatal("prefetch did not finish")
	}
	got, err := p.store.LoadDescriptor("vacuum")
	require.NoError(t, err)
	require.Equal(t, "cooldown", got.DetectionCooldownConfigField)
	cfg, err := p.LoadJobTypeConfig("vacuum")
	require.NoError(t, err)
	require.Equal(t, int64(600), cfg.WorkerConfigValues["cooldown"].GetInt64Value())
}

func TestRunDetectionIncludesLatestSuccessfulRun(t *testing.T) {
	pluginSvc, err := New(Options{})
	if err != nil {
		t.Fatalf("New plugin error: %v", err)
	}
	defer pluginSvc.Shutdown()

	jobType := "vacuum"
	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
		WorkerId: "worker-a",
		Capabilities: []*plugin_pb.JobTypeCapability{
			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
		},
	})
	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
	pluginSvc.putSession(session)

	oldSuccess := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
	latestSuccess := time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC)
	if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{Outcome: RunOutcomeSuccess, CompletedAt: timeToPtr(oldSuccess)}); err != nil {
		t.Fatalf("AppendRunRecord old success: %v", err)
	}
	if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{Outcome: RunOutcomeError, CompletedAt: timeToPtr(latestSuccess.Add(2 * time.Hour))}); err != nil {
		t.Fatalf("AppendRunRecord error run: %v", err)
	}
	if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{Outcome: RunOutcomeSuccess, CompletedAt: timeToPtr(latestSuccess)}); err != nil {
		t.Fatalf("AppendRunRecord latest success: %v", err)
	}

	resultCh := make(chan error, 1)
	go func() {
		_, runErr := pluginSvc.RunDetection(context.Background(), jobType, &plugin_pb.ClusterContext{}, 10)
		resultCh <- runErr
	}()

	message := <-session.outgoing
	detectRequest := message.GetRunDetectionRequest()
	if detectRequest == nil {
		t.Fatalf("expected run detection request message")
	}
	if detectRequest.LastSuccessfulRun == nil {
		t.Fatalf("expected last_successful_run to be set")
	}
	if got := detectRequest.LastSuccessfulRun.AsTime().UTC(); !got.Equal(latestSuccess) {
		t.Fatalf("unexpected last_successful_run, got=%s want=%s", got, latestSuccess)
	}

	pluginSvc.handleDetectionComplete("worker-a", &plugin_pb.DetectionComplete{
		RequestId: message.RequestId,
		JobType:   jobType,
		Success:   true,
	})

	if runErr := <-resultCh; runErr != nil {
		t.Fatalf("RunDetection error: %v", runErr)
	}
}

func TestRunDetectionOmitsLastSuccessfulRunWhenNoSuccessHistory(t *testing.T) {
	pluginSvc, err := New(Options{})
	if err != nil {
		t.Fatalf("New plugin error: %v", err)
	}
	defer pluginSvc.Shutdown()

	jobType := "vacuum"
	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
		WorkerId: "worker-a",
		Capabilities: []*plugin_pb.JobTypeCapability{
			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
		},
	})
	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
	pluginSvc.putSession(session)

	if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{
		Outcome:     RunOutcomeError,
		CompletedAt: timeToPtr(time.Date(2026, 2, 10, 0, 0, 0, 0, time.UTC)),
	}); err != nil {
		t.Fatalf("AppendRunRecord error run: %v", err)
	}

	resultCh := make(chan error, 1)
	go func() {
		_, runErr := pluginSvc.RunDetection(context.Background(), jobType, &plugin_pb.ClusterContext{}, 10)
		resultCh <- runErr
	}()

	message := <-session.outgoing
	detectRequest := message.GetRunDetectionRequest()
	if detectRequest == nil {
		t.Fatalf("expected run detection request message")
	}
	if detectRequest.LastSuccessfulRun != nil {
		t.Fatalf("expected last_successful_run to be nil when no success history")
	}

	pluginSvc.handleDetectionComplete("worker-a", &plugin_pb.DetectionComplete{
		RequestId: message.RequestId,
		JobType:   jobType,
		Success:   true,
	})

	if runErr := <-resultCh; runErr != nil {
		t.Fatalf("RunDetection error: %v", runErr)
	}
}

func TestRunDetectionWithReportCapturesDetectionActivities(t *testing.T) {
	pluginSvc, err := New(Options{})
	if err != nil {
		t.Fatalf("New plugin error: %v", err)
	}
	defer pluginSvc.Shutdown()

	jobType := "vacuum"
	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
		WorkerId: "worker-a",
		Capabilities: []*plugin_pb.JobTypeCapability{
			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
		},
	})
	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
	pluginSvc.putSession(session)

	reportCh := make(chan *DetectionReport, 1)
	errCh := make(chan error, 1)
	go func() {
		report, runErr := pluginSvc.RunDetectionWithReport(context.Background(), jobType, &plugin_pb.ClusterContext{}, 10)
		reportCh <- report
		errCh <- runErr
	}()

	message := <-session.outgoing
	requestID := message.GetRequestId()
	if requestID == "" {
		t.Fatalf("expected request id in detection request")
	}

	pluginSvc.handleDetectionProposals("worker-a", &plugin_pb.DetectionProposals{
		RequestId: requestID,
		JobType:   jobType,
		Proposals: []*plugin_pb.JobProposal{
			{
				ProposalId: "proposal-1",
				JobType:    jobType,
				Summary:    "vacuum proposal",
				Detail:     "based on garbage ratio",
			},
		},
	})
	pluginSvc.handleDetectionComplete("worker-a", &plugin_pb.DetectionComplete{
		RequestId:      requestID,
		JobType:        jobType,
		Success:        true,
		TotalProposals: 1,
	})

	report := <-reportCh
	if report == nil {
		t.Fatalf("expected detection report")
	}
	if report.RequestID == "" {
		t.Fatalf("expected detection report request id")
	}
	if report.WorkerID != "worker-a" {
		t.Fatalf("expected worker-a, got %q", report.WorkerID)
	}
	if len(report.Proposals) != 1 {
		t.Fatalf("expected one proposal in report, got %d", len(report.Proposals))
	}
	if runErr := <-errCh; runErr != nil {
		t.Fatalf("RunDetectionWithReport error: %v", runErr)
	}

	activities := pluginSvc.ListActivities(jobType, 0)
	stages := map[string]bool{}
	for _, activity := range activities {
		if activity.RequestID != report.RequestID {
			continue
		}
		stages[activity.Stage] = true
	}
	if !stages["requested"] || !stages["proposal"] || !stages["completed"] {
		t.Fatalf("expected requested/proposal/completed activities, got stages=%v", stages)
	}
}

func TestRunDetectionAdminScriptUsesLastCompletedRun(t *testing.T) {
	pluginSvc, err := New(Options{})
	if err != nil {
		t.Fatalf("New plugin error: %v", err)
	}
	defer pluginSvc.Shutdown()

	jobType := "admin_script"
	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
		WorkerId: "worker-admin-script",
		Capabilities: []*plugin_pb.JobTypeCapability{
			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
		},
	})
	session := &streamSession{workerID: "worker-admin-script", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
	pluginSvc.putSession(session)

	successCompleted := time.Date(2026, 2, 1, 10, 0, 0, 0, time.UTC)
	errorCompleted := successCompleted.Add(45 * time.Minute)
	if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{
		Outcome:     RunOutcomeSuccess,
		CompletedAt: timeToPtr(successCompleted),
	}); err != nil {
		t.Fatalf("AppendRunRecord success run: %v", err)
	}
	if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{
		Outcome:     RunOutcomeError,
		CompletedAt: timeToPtr(errorCompleted),
	}); err != nil {
		t.Fatalf("AppendRunRecord error run: %v", err)
	}

	errCh := make(chan error, 1)
	go func() {
		_, runErr := pluginSvc.RunDetection(context.Background(), jobType, &plugin_pb.ClusterContext{}, 10)
		errCh <- runErr
	}()

	message := <-session.outgoing
	detectRequest := message.GetRunDetectionRequest()
	if detectRequest == nil {
		t.Fatalf("expected run detection request message")
	}
	if detectRequest.LastSuccessfulRun == nil {
		t.Fatalf("expected last_successful_run to be set")
	}
	if got := detectRequest.LastSuccessfulRun.AsTime().UTC(); !got.Equal(errorCompleted) {
		t.Fatalf("unexpected last_successful_run, got=%s want=%s", got, errorCompleted)
	}

	pluginSvc.handleDetectionComplete("worker-admin-script", &plugin_pb.DetectionComplete{
		RequestId: message.RequestId,
		JobType:   jobType,
		Success:   true,
	})

	if runErr := <-errCh; runErr != nil {
		t.Fatalf("RunDetection error: %v", runErr)
	}
}