seaweedfs/weed/admin/plugin/orphaned_execution.go

package plugin

import (
	"fmt"
	"strings"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)

// orphanedExecutionMisses is how many itemized heartbeats in a row from a
// newer stream of the worker must leave a dispatched job out before the
// admin fails it. The second rides out a heartbeat the worker built while
// it was still picking the job back up.
const orphanedExecutionMisses = 2

// pendingExecutionState is an execution request waiting for its JobCompleted.
type pendingExecutionState struct {
	completed chan *plugin_pb.JobCompleted
	jobID     string
	jobType   string
	// session is the stream the request went out on.
	session *streamSession
	misses  int
	failed  bool
	// completionSent is set once a completion is handed to the dispatcher,
	// which may already have taken it from the buffer.
	completionSent bool
}

// failExecutionsMissingFromHeartbeat fails the executions sent on an earlier
// stream of the heartbeat's worker that the worker no longer lists. A worker
// that restarted comes back under the same id without the old process's
// jobs, and nothing else ends such a dispatch before its execution timeout,
// so its slot stays reserved all that time. Jobs sent on the heartbeat's own
// stream are left alone, since a worker counts a job before it lists it, and
// so is everything while the heartbeat counts slots it does not itemize,
// since a missing id then proves nothing.
func (r *Plugin) failExecutionsMissingFromHeartbeat(session *streamSession, heartbeat *plugin_pb.WorkerHeartbeat) {
	if session == nil || heartbeat == nil {
		return
	}
	itemized := 0
	listed := make(map[string]struct{}, len(heartbeat.GetRunningWork()))
	for _, work := range heartbeat.GetRunningWork() {
		if work.GetKind() == plugin_pb.WorkKind_WORK_KIND_EXECUTION {
			listed[strings.TrimSpace(work.GetWorkId())] = struct{}{}
			itemized++
		}
	}
	itemizesAll := int(heartbeat.GetExecutionSlotsUsed()) <= itemized

	var orphaned []activeJobSnapshot
	r.pendingExecutionMu.Lock()
	for requestID, pending := range r.pendingExecution {
		if pending.failed || pending.session == nil || pending.session == session || pending.session.workerID != session.workerID {
			continue
		}
		if _, ok := listed[strings.TrimSpace(pending.jobID)]; ok {
			pending.misses = 0
			continue
		}
		if !itemizesAll {
			continue
		}
		if pending.completionSent {
			continue
		}
		pending.misses++
		if pending.misses < orphanedExecutionMisses {
			continue
		}
		pending.failed = true
		orphaned = append(orphaned, activeJobSnapshot{
			jobID:     pending.jobID,
			jobType:   pending.jobType,
			workerID:  session.workerID,
			requestID: requestID,
		})
	}
	r.pendingExecutionMu.Unlock()

	for _, snap := range orphaned {
		reason := fmt.Sprintf("worker %s re-registered without job %s", snap.workerID, snap.jobID)
		glog.Warningf("Plugin failing %s request %s: %s", snap.jobType, snap.requestID, reason)
		// off the stream's receive loop: the cancel may wait on a full buffer
		go r.expireActiveJob(snap, reason, time.Now().UTC())
	}
}