seaweedfs/weed/admin/plugin/worker_job_queues.go

package plugin

import (
	"sync"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)

// queuedJob is one scheduled spec parked on a home worker's admin-side queue.
type queuedJob struct {
	job  *plugin_pb.JobSpec
	home string
}

// workerJobQueues holds per-worker FIFOs for one dispatch batch.
// Home assignment is done once up front; run goroutines only pop/steal.
// leftover is the homeless FIFO: replica hostnames with no currently
// registered worker. Orphan grace does not apply here.
type workerJobQueues struct {
	mu           sync.Mutex
	q            map[string][]*queuedJob
	leftover     []*queuedJob
	remaining    int
	missingSince map[string]time.Time
	homes        map[string]struct{}
}

func newWorkerJobQueues() *workerJobQueues {
	return &workerJobQueues{
		q:            make(map[string][]*queuedJob),
		missingSince: make(map[string]time.Time),
		homes:        make(map[string]struct{}),
	}
}

func (qs *workerJobQueues) hasHome(workerID string) bool {
	if qs == nil || workerID == "" {
		return false
	}
	_, ok := qs.homes[workerID]
	return ok
}

func (qs *workerJobQueues) push(home string, job *plugin_pb.JobSpec) {
	if qs == nil || job == nil {
		return
	}
	item := &queuedJob{job: job, home: home}
	qs.mu.Lock()
	defer qs.mu.Unlock()
	qs.pushLocked(item, false)
}

func (qs *workerJobQueues) pushFront(item *queuedJob) {
	if qs == nil || item == nil {
		return
	}
	qs.mu.Lock()
	defer qs.mu.Unlock()
	qs.pushLocked(item, true)
}

func (qs *workerJobQueues) pushLocked(item *queuedJob, front bool) {
	qs.remaining++
	if item.home == "" {
		if front {
			qs.leftover = append([]*queuedJob{item}, qs.leftover...)
			return
		}
		qs.leftover = append(qs.leftover, item)
		return
	}
	if front {
		qs.q[item.home] = append([]*queuedJob{item}, qs.q[item.home]...)
		return
	}
	qs.q[item.home] = append(qs.q[item.home], item)
}

func (qs *workerJobQueues) isEmpty() bool {
	if qs == nil {
		return true
	}
	qs.mu.Lock()
	defer qs.mu.Unlock()
	return qs.remaining == 0
}

func (qs *workerJobQueues) queueLen(workerID string) int {
	if qs == nil {
		return 0
	}
	qs.mu.Lock()
	defer qs.mu.Unlock()
	return len(qs.q[workerID])
}

func (qs *workerJobQueues) leftoverLen() int {
	if qs == nil {
		return 0
	}
	qs.mu.Lock()
	defer qs.mu.Unlock()
	return len(qs.leftover)
}

func (qs *workerJobQueues) drain() []*queuedJob {
	if qs == nil {
		return nil
	}
	qs.mu.Lock()
	defer qs.mu.Unlock()
	out := make([]*queuedJob, 0, qs.remaining)
	for id, qq := range qs.q {
		out = append(out, qq...)
		qs.q[id] = nil
	}
	out = append(out, qs.leftover...)
	qs.leftover = nil
	qs.remaining = 0
	return out
}

func popFront(qq []*queuedJob) (*queuedJob, []*queuedJob) {
	item := qq[0]
	if len(qq) == 1 {
		return item, nil
	}
	return item, qq[1:]
}

// assignJobsToWorkerQueues parks each job on the least-loaded replica-host
// worker. A home exists only if a currently registered worker's -address
// hostname matches a replica host. Otherwise the job is homeless: global
// leftover FIFO, no fake home, no K, no orphan grace. Single-threaded.
func (r *Plugin) assignJobsToWorkerQueues(
	jobType string,
	jobs []*plugin_pb.JobSpec,
	_ schedulerPolicy,
) *workerJobQueues {
	qs := newWorkerJobQueues()
	executors, err := r.registry.ListExecutors(jobType)
	lane := JobTypeLane(jobType)
	queued := make(map[string]int, len(executors))

	for _, job := range jobs {
		if job == nil {
			continue
		}
		if err != nil || len(executors) == 0 {
			qs.push("", job)
			continue
		}
		pref, _ := partitionByPreferredHost(executors, JobReplicaHosts(job))
		if len(pref) == 0 {
			// Never registered for that hostname: homeless, not an orphan.
			qs.push("", job)
			continue
		}
		for _, e := range pref {
			if e != nil && e.WorkerID != "" {
				qs.homes[e.WorkerID] = struct{}{}
			}
		}
		sortExecutorsByLoad(pref, func(e *WorkerSession) int {
			return r.executorReservationLoad(e, lane) + queued[e.WorkerID]
		})
		home := pref[0].WorkerID
		qs.push(home, job)
		queued[home]++
		r.bindQueuedJobWorker(job, home)
	}
	return qs
}

// takeQueuedJob pops a job for a free worker.
// Home worker (hostname matched a replica / has a home-queue identity):
//  1. own home queue
//  2. steal from another home if len>=K, or orphan past OrphanRehomeGrace
//  3. homeless leftover FIFO
// No-home worker (no matching replica, no home-queue identity):
//  1. homeless leftover FIFO
//  2. only if leftover empty: steal (K / orphan), same rules
func (r *Plugin) takeQueuedJob(
	qs *workerJobQueues,
	jobType string,
	policy schedulerPolicy,
) (item *queuedJob, targetID string) {
	if qs == nil {
		return nil, ""
	}
	executors, err := r.registry.ListExecutors(jobType)
	if err != nil || len(executors) == 0 {
		return nil, ""
	}
	lane := JobTypeLane(jobType)
	pin := policy.LocalQueuePin
	defer func() {
		if item != nil && targetID != "" {
			r.bindQueuedJobWorker(item.job, targetID)
		}
	}()

	qs.mu.Lock()
	defer qs.mu.Unlock()
	if qs.remaining == 0 {
		return nil, ""
	}

	var bestHome *WorkerSession
	bestLoad := 0
	for _, exec := range executors {
		if len(qs.q[exec.WorkerID]) == 0 {
			continue
		}
		limit := schedulerWorkerExecutionLimit(exec, jobType, policy)
		if limit <= 0 {
			continue
		}
		load := r.executorReservationLoad(exec, lane)
		if load >= limit {
			continue
		}
		if bestHome == nil || load < bestLoad || (load == bestLoad && exec.WorkerID < bestHome.WorkerID) {
			bestHome = exec
			bestLoad = load
		}
	}
	if bestHome != nil {
		id := bestHome.WorkerID
		item, qs.q[id] = popFront(qs.q[id])
		qs.remaining--
		return item, id
	}

	var thief *WorkerSession
	for _, exec := range executors {
		limit := schedulerWorkerExecutionLimit(exec, jobType, policy)
		if limit <= 0 {
			continue
		}
		if r.executorReservationLoad(exec, lane) >= limit {
			continue
		}
		if thief == nil || exec.WorkerID < thief.WorkerID {
			thief = exec
		}
	}
	if thief == nil {
		return nil, ""
	}

	takeLeftover := func() (*queuedJob, string) {
		if len(qs.leftover) == 0 {
			return nil, ""
		}
		item, qs.leftover = popFront(qs.leftover)
		qs.remaining--
		return item, thief.WorkerID
	}
	if !qs.hasHome(thief.WorkerID) {
		if item, id := takeLeftover(); item != nil {
			return item, id
		}
	}

	live := make(map[string]struct{}, len(executors))
	for _, exec := range executors {
		live[exec.WorkerID] = struct{}{}
		delete(qs.missingSince, exec.WorkerID)
	}

	now := time.Now()
	grace := policy.OrphanRehomeGrace
	victimID := ""
	victimLen := 0
	victimOrphan := false
	for id, qq := range qs.q {
		if len(qq) == 0 {
			delete(qs.missingSince, id)
			continue
		}
		_, isLive := live[id]
		orphan := !isLive
		if isLive {
			if pin > 0 && len(qq) < pin {
				continue
			}
		} else {
			if _, ok := qs.missingSince[id]; !ok {
				qs.missingSince[id] = now
			}
			if grace > 0 && now.Sub(qs.missingSince[id]) < grace {
				continue
			}
		}
		better := victimID == ""
		if !better && orphan != victimOrphan {
			better = orphan
		}
		if !better && orphan == victimOrphan {
			better = len(qq) > victimLen || (len(qq) == victimLen && id < victimID)
		}
		if better {
			victimID = id
			victimLen = len(qq)
			victimOrphan = orphan
		}
	}
	if victimID != "" {
		item, qs.q[victimID] = popFront(qs.q[victimID])
		qs.remaining--
		return item, thief.WorkerID
	}
	return takeLeftover()
}