mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 10:54:08 +01:00
EC worker: pin jobs to replica-host workers, late-bind local source (#11718)
* feat: prefer same-host worker and local EC replica Admin executor pick ranked idle workers only, so a job whose volume lived on full-03 could still run on full-01 and pull the whole .dat across the LAN. Rank workers whose Hello host matches any replica first (fallback to others if the local worker is full). Detection named Sources[0] / the lowest server id; execute then SyncAndSelectBestReplica and could switch the copy source to another rack. After sync, if a replica on this machine has a complete file count, copy from that URL so affinity and pull stay on one host. * feat: pin scheduler jobs to replica-host workers until load K Reserve prefers the least-loaded replica-host worker and waits while that load is below 5 instead of spilling to a free remote worker. K=0 restores immediate spill. Waterfill uses reservation load so four idle preferred workers each get a job. * feat: park scheduled jobs on per-worker admin queues Assign a detection batch to the least-loaded replica-host worker in one pass, then let ExecutionConcurrency goroutines pop a home-runnable job or steal when that home queue reaches K. Dispatch no longer holds a job in reserve-backoff, so a free worker is not blocked by another home's pin. Tracked jobs stay pending/queued and record the home worker for the admin UI. * fix: rehome queued jobs when the home worker dies A live home still pins below K. A home missing from live executors is stolen immediately onto a free worker so a short orphaned queue does not sit until the dispatch window ends. * feat: wait before stealing a dead home worker queue Orphaned queues are held for 30s so a worker reconnect blink does not dump jobs onto other machines. After the grace, steal without K as before. Zero disables the wait. A home that comes back during grace keeps its queue. * fix: late-bind EC from to worker hostname After replicas are consistent (equal FileCount), rebind encode source to a volume server on the same hostname as worker -address. Any port on that host is local. Diverged FileCount still uses the max replica. * feat: assign homeless jobs to global FIFO Jobs whose replica hostname has no registered worker skip pin and orphan grace. Free workers take own home, then steal at K/orphan, then leftover FIFO. * fix: take homeless FIFO before steal for no-home workers Workers without a replica-host match take leftover first; steal (K/orphan) only when FIFO is empty. Home workers keep own-queue, steal, then leftover. --------- Co-authored-by: Chepurnov Simon <195125934+CHEPURNOV-S-S@users.noreply.github.com>
This commit is contained in:
1 parent
492e700d1f
commit
624a13ecc6
16 files changed
+1601
-141
No files matched your search
@@ -0,0 +1,66 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
// JobReplicaHosts are the volume-replica hosts a job will read from.
|
||||
// Used so executor pick prefers a worker on one of those machines.
|
||||
// Sources with ShardIds are EC-cleanup rows, not encode replicas.
|
||||
func JobReplicaHosts(job *plugin_pb.JobSpec) []string {
|
||||
if job == nil {
|
||||
return nil
|
||||
}
|
||||
var addrs []string
|
||||
addrs = append(addrs, pluginworker.ReadStringListConfig(job.Parameters, "replica_servers")...)
|
||||
if src := strings.TrimSpace(pluginworker.ReadStringConfig(job.Parameters, "source_server", "")); src != "" {
|
||||
addrs = append(addrs, src)
|
||||
}
|
||||
if payload := pluginworker.ReadBytesConfig(job.Parameters, "task_params_pb"); len(payload) > 0 {
|
||||
params := &worker_pb.TaskParams{}
|
||||
if err := proto.Unmarshal(payload, params); err == nil {
|
||||
for _, s := range params.GetSources() {
|
||||
if s == nil || s.Node == "" || len(s.ShardIds) > 0 {
|
||||
continue
|
||||
}
|
||||
addrs = append(addrs, s.Node)
|
||||
}
|
||||
}
|
||||
}
|
||||
return util.HostKeysFromAddresses(addrs)
|
||||
}
|
||||
|
||||
// partitionByPreferredHost splits candidates into (host matches, others).
|
||||
// No preferred hosts, or no matches, returns (nil, all) so callers keep
|
||||
// the existing slots+round-robin path.
|
||||
func partitionByPreferredHost(candidates []*WorkerSession, preferredHosts []string) (pref, rest []*WorkerSession) {
|
||||
want := util.HostKeysFromAddresses(preferredHosts)
|
||||
if len(want) == 0 || len(candidates) == 0 {
|
||||
return nil, candidates
|
||||
}
|
||||
set := make(map[string]struct{}, len(want))
|
||||
for _, h := range want {
|
||||
set[h] = struct{}{}
|
||||
}
|
||||
for _, c := range candidates {
|
||||
if c == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := set[util.HostKey(c.Address)]; ok {
|
||||
pref = append(pref, c)
|
||||
continue
|
||||
}
|
||||
rest = append(rest, c)
|
||||
}
|
||||
if len(pref) == 0 {
|
||||
return nil, candidates
|
||||
}
|
||||
return pref, rest
|
||||
}
|
||||
@@ -615,7 +615,7 @@ func (r *Plugin) ExecuteJob(
|
||||
releaseGate := r.acquireDetectionExecutionGate(jobType, true)
|
||||
defer releaseGate()
|
||||
|
||||
executor, err := r.registry.PickExecutor(jobType)
|
||||
executor, err := r.registry.PickExecutor(jobType, JobReplicaHosts(job)...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -887,6 +887,37 @@ func (r *Plugin) trackExecutionQueued(job *plugin_pb.JobSpec) {
|
||||
})
|
||||
}
|
||||
|
||||
// bindQueuedJobWorker records the home worker while the job stays pending/queued.
|
||||
func (r *Plugin) bindQueuedJobWorker(job *plugin_pb.JobSpec, workerID string) {
|
||||
if job == nil || strings.TrimSpace(job.JobId) == "" || strings.TrimSpace(workerID) == "" {
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
r.jobsMu.Lock()
|
||||
tracked := r.jobs[job.JobId]
|
||||
if tracked == nil {
|
||||
r.jobsMu.Unlock()
|
||||
return
|
||||
}
|
||||
if isTerminalTrackedJobState(tracked.State) {
|
||||
r.jobsMu.Unlock()
|
||||
return
|
||||
}
|
||||
tracked.WorkerID = workerID
|
||||
tracked.Message = "queued on worker " + workerID
|
||||
tracked.UpdatedAt = timeToPtr(now)
|
||||
snap := cloneTrackedJob(*tracked)
|
||||
r.dirtyJobs = true
|
||||
r.jobsMu.Unlock()
|
||||
|
||||
r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) {
|
||||
detail.WorkerID = snap.WorkerID
|
||||
detail.Message = snap.Message
|
||||
detail.UpdatedAt = snap.UpdatedAt
|
||||
})
|
||||
}
|
||||
|
||||
func (r *Plugin) cancelQueuedJob(job *plugin_pb.JobSpec, cause error) {
|
||||
reason := "job canceled"
|
||||
if cause != nil {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -28,12 +29,18 @@ const (
|
||||
defaultScheduledMaxResults int32 = 1000
|
||||
defaultScheduledExecutionConcurrency = 1
|
||||
defaultScheduledPerWorkerConcurrency = 1
|
||||
maxScheduledExecutionConcurrency = 128
|
||||
defaultScheduledRetryBackoff = 5 * time.Second
|
||||
defaultClusterContextTimeout = 10 * time.Second
|
||||
defaultWaitingBacklogFloor = 8
|
||||
defaultWaitingBacklogMultiplier = 4
|
||||
maxEstimatedRuntimeCap = 8 * time.Hour
|
||||
// Hold a job on replica-host workers until their reservation load
|
||||
// reaches this; 0 disables the pin and spills immediately.
|
||||
defaultLocalQueuePin = 5
|
||||
// Wait this long after a home worker disappears before stealing
|
||||
// its queue; 0 steals immediately. Covers a reconnect blink.
|
||||
defaultOrphanRehomeGrace = 30 * time.Second
|
||||
maxScheduledExecutionConcurrency = 128
|
||||
defaultScheduledRetryBackoff = 5 * time.Second
|
||||
defaultClusterContextTimeout = 10 * time.Second
|
||||
defaultWaitingBacklogFloor = 8
|
||||
defaultWaitingBacklogMultiplier = 4
|
||||
maxEstimatedRuntimeCap = 8 * time.Hour
|
||||
// How long started jobs may drain past the run window close.
|
||||
scheduledExecutionDrainGrace = 30 * time.Minute
|
||||
)
|
||||
@@ -49,6 +56,12 @@ type schedulerPolicy struct {
|
||||
PerWorkerConcurrency int
|
||||
RetryLimit int
|
||||
ExecutorReserveBackoff time.Duration
|
||||
// LocalQueuePin is K: do not spill off replica-host workers while
|
||||
// min(load(pref)) < K. Zero is the old immediate-spill path.
|
||||
LocalQueuePin int
|
||||
// OrphanRehomeGrace is how long a missing home's queue is held
|
||||
// before steal-without-K. Zero steals as soon as the worker is gone.
|
||||
OrphanRehomeGrace time.Duration
|
||||
}
|
||||
|
||||
// laneSchedulerLoop is the main scheduling goroutine for a single lane.
|
||||
@@ -466,6 +479,8 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
|
||||
PerWorkerConcurrency: int(adminRuntime.PerWorkerExecutionConcurrency),
|
||||
RetryLimit: int(adminRuntime.RetryLimit),
|
||||
ExecutorReserveBackoff: 200 * time.Millisecond,
|
||||
LocalQueuePin: defaultLocalQueuePin,
|
||||
OrphanRehomeGrace: defaultOrphanRehomeGrace,
|
||||
}
|
||||
|
||||
if policy.DetectionInterval < r.schedulerTick {
|
||||
@@ -496,6 +511,9 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
|
||||
if policy.ExecutionTimeout < defaultScheduledExecutionTimeout {
|
||||
policy.ExecutionTimeout = defaultScheduledExecutionTimeout
|
||||
}
|
||||
if policy.LocalQueuePin < 0 {
|
||||
policy.LocalQueuePin = 0
|
||||
}
|
||||
|
||||
return policy, true, nil
|
||||
}
|
||||
@@ -813,19 +831,22 @@ func (r *Plugin) dispatchScheduledProposals(
|
||||
// whole dispatch lets manual shell operations interleave between jobs.
|
||||
jobsRequireLock := LaneRequiresLock(JobTypeLane(jobType))
|
||||
|
||||
jobQueue := make(chan *plugin_pb.JobSpec, len(proposals))
|
||||
jobs := make([]*plugin_pb.JobSpec, 0, len(proposals))
|
||||
for index, proposal := range proposals {
|
||||
job := buildScheduledJobSpec(jobType, proposal, index)
|
||||
r.trackExecutionQueued(job)
|
||||
select {
|
||||
case <-r.shutdownCh:
|
||||
close(jobQueue)
|
||||
return 0, 0, 0
|
||||
for _, job := range jobs {
|
||||
r.cancelQueuedJob(job, errSchedulerShutdown)
|
||||
}
|
||||
return 0, 0, len(jobs)
|
||||
default:
|
||||
jobQueue <- job
|
||||
}
|
||||
job := buildScheduledJobSpec(jobType, proposal, index)
|
||||
r.trackExecutionQueued(job)
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
close(jobQueue)
|
||||
|
||||
queues := r.assignJobsToWorkerQueues(jobType, jobs, policy)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var statsMu sync.Mutex
|
||||
@@ -842,130 +863,110 @@ func (r *Plugin) dispatchScheduledProposals(
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
jobLoop:
|
||||
for job := range jobQueue {
|
||||
for {
|
||||
select {
|
||||
case <-r.shutdownCh:
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
r.cancelQueuedJob(job, ctx.Err())
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
return
|
||||
}
|
||||
if queues.isEmpty() {
|
||||
return
|
||||
}
|
||||
|
||||
item, targetID := r.takeQueuedJob(queues, jobType, policy)
|
||||
if item == nil {
|
||||
if queues.isEmpty() {
|
||||
return
|
||||
}
|
||||
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
queues.pushFront(item)
|
||||
return
|
||||
}
|
||||
|
||||
executor, release, reserveErr := r.reserveExecutorByID(jobType, policy, targetID)
|
||||
if reserveErr != nil {
|
||||
queues.pushFront(item)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-r.shutdownCh:
|
||||
return
|
||||
default:
|
||||
var releaseJobLock func()
|
||||
if jobsRequireLock {
|
||||
var lockErr error
|
||||
releaseJobLock, lockErr = r.acquireAdminLock(fmt.Sprintf("plugin scheduler %s job %s", jobType, item.job.JobId))
|
||||
if lockErr != nil {
|
||||
release()
|
||||
statsMu.Lock()
|
||||
errorCount++
|
||||
statsMu.Unlock()
|
||||
r.appendActivity(JobActivity{
|
||||
JobID: item.job.JobId,
|
||||
JobType: item.job.JobType,
|
||||
Source: "admin_scheduler",
|
||||
Message: fmt.Sprintf("scheduled execution lock failed: %v", lockErr),
|
||||
Stage: "failed",
|
||||
OccurredAt: timeToPtr(time.Now().UTC()),
|
||||
})
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
r.cancelQueuedJob(job, ctx.Err())
|
||||
releaseJobLock()
|
||||
release()
|
||||
queues.pushFront(item)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
err := r.executeScheduledJobWithExecutor(ctx, executor, item.job, clusterContext, policy)
|
||||
if releaseJobLock != nil {
|
||||
releaseJobLock()
|
||||
}
|
||||
release()
|
||||
if errors.Is(err, errExecutorAtCapacity) {
|
||||
queues.pushFront(item)
|
||||
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
if ctx.Err() != nil || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
r.cancelQueuedJob(item.job, err)
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
continue jobLoop
|
||||
}
|
||||
|
||||
executor, release, reserveErr := r.reserveScheduledExecutor(ctx, jobType, policy)
|
||||
if reserveErr != nil {
|
||||
if ctx.Err() != nil {
|
||||
r.cancelQueuedJob(job, ctx.Err())
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
continue jobLoop
|
||||
}
|
||||
statsMu.Lock()
|
||||
errorCount++
|
||||
statsMu.Unlock()
|
||||
r.appendActivity(JobActivity{
|
||||
JobType: jobType,
|
||||
Source: "admin_scheduler",
|
||||
Message: fmt.Sprintf("scheduled execution reservation failed: %v", reserveErr),
|
||||
Stage: "failed",
|
||||
OccurredAt: timeToPtr(time.Now().UTC()),
|
||||
})
|
||||
break
|
||||
}
|
||||
|
||||
var releaseJobLock func()
|
||||
if jobsRequireLock {
|
||||
var lockErr error
|
||||
releaseJobLock, lockErr = r.acquireAdminLock(fmt.Sprintf("plugin scheduler %s job %s", jobType, job.JobId))
|
||||
if lockErr != nil {
|
||||
release()
|
||||
statsMu.Lock()
|
||||
errorCount++
|
||||
statsMu.Unlock()
|
||||
r.appendActivity(JobActivity{
|
||||
JobID: job.JobId,
|
||||
JobType: job.JobType,
|
||||
Source: "admin_scheduler",
|
||||
Message: fmt.Sprintf("scheduled execution lock failed: %v", lockErr),
|
||||
Stage: "failed",
|
||||
OccurredAt: timeToPtr(time.Now().UTC()),
|
||||
})
|
||||
break
|
||||
}
|
||||
// The lock may have been held by an operator for a
|
||||
// while; re-check the execution window.
|
||||
if ctx.Err() != nil {
|
||||
releaseJobLock()
|
||||
release()
|
||||
r.cancelQueuedJob(job, ctx.Err())
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
continue jobLoop
|
||||
}
|
||||
}
|
||||
|
||||
err := r.executeScheduledJobWithExecutor(ctx, executor, job, clusterContext, policy)
|
||||
if releaseJobLock != nil {
|
||||
releaseJobLock()
|
||||
}
|
||||
release()
|
||||
if errors.Is(err, errExecutorAtCapacity) {
|
||||
r.trackExecutionQueued(job)
|
||||
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
if ctx.Err() != nil || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
r.cancelQueuedJob(job, err)
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
continue jobLoop
|
||||
}
|
||||
statsMu.Lock()
|
||||
errorCount++
|
||||
statsMu.Unlock()
|
||||
r.appendActivity(JobActivity{
|
||||
JobID: job.JobId,
|
||||
JobType: job.JobType,
|
||||
Source: "admin_scheduler",
|
||||
Message: fmt.Sprintf("scheduled execution failed: %v", err),
|
||||
Stage: "failed",
|
||||
OccurredAt: timeToPtr(time.Now().UTC()),
|
||||
})
|
||||
break
|
||||
}
|
||||
|
||||
statsMu.Lock()
|
||||
successCount++
|
||||
errorCount++
|
||||
statsMu.Unlock()
|
||||
break
|
||||
r.appendActivity(JobActivity{
|
||||
JobID: item.job.JobId,
|
||||
JobType: item.job.JobType,
|
||||
Source: "admin_scheduler",
|
||||
Message: fmt.Sprintf("scheduled execution failed: %v", err),
|
||||
Stage: "failed",
|
||||
OccurredAt: timeToPtr(time.Now().UTC()),
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
statsMu.Lock()
|
||||
successCount++
|
||||
statsMu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -976,8 +977,8 @@ func (r *Plugin) dispatchScheduledProposals(
|
||||
if drainErr == nil {
|
||||
drainErr = errSchedulerShutdown
|
||||
}
|
||||
for job := range jobQueue {
|
||||
r.cancelQueuedJob(job, drainErr)
|
||||
for _, item := range queues.drain() {
|
||||
r.cancelQueuedJob(item.job, drainErr)
|
||||
canceledCount++
|
||||
}
|
||||
|
||||
@@ -988,6 +989,7 @@ func (r *Plugin) reserveScheduledExecutor(
|
||||
ctx context.Context,
|
||||
jobType string,
|
||||
policy schedulerPolicy,
|
||||
preferredHosts []string,
|
||||
) (*WorkerSession, func(), error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
@@ -1015,7 +1017,7 @@ func (r *Plugin) reserveScheduledExecutor(
|
||||
return nil, nil, fmt.Errorf("timed out waiting for executor capacity for %s", jobType)
|
||||
}
|
||||
|
||||
executors, err := r.registry.ListExecutors(jobType)
|
||||
executors, err := r.registry.ListExecutors(jobType, preferredHosts...)
|
||||
if err != nil {
|
||||
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
|
||||
if ctx.Err() != nil {
|
||||
@@ -1026,11 +1028,19 @@ func (r *Plugin) reserveScheduledExecutor(
|
||||
continue
|
||||
}
|
||||
|
||||
for _, executor := range executors {
|
||||
release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy)
|
||||
if !ok {
|
||||
continue
|
||||
pref, rest := partitionByPreferredHost(executors, preferredHosts)
|
||||
lane := JobTypeLane(jobType)
|
||||
loadOf := func(e *WorkerSession) int { return r.executorReservationLoad(e, lane) }
|
||||
|
||||
if len(pref) > 0 {
|
||||
sortExecutorsByLoad(pref, loadOf)
|
||||
if executor, release, ok := r.tryReserveFirst(pref, jobType, policy); ok {
|
||||
return executor, release, nil
|
||||
}
|
||||
}
|
||||
|
||||
sortExecutorsByLoad(rest, loadOf)
|
||||
if executor, release, ok := r.tryReserveFirst(rest, jobType, policy); ok {
|
||||
return executor, release, nil
|
||||
}
|
||||
|
||||
@@ -1051,6 +1061,75 @@ func (r *Plugin) tryReserveExecutorCapacity(
|
||||
return r.tryReserveExecutorCapacityForLane(executor, jobType, policy, JobTypeLane(jobType))
|
||||
}
|
||||
|
||||
func (r *Plugin) tryReserveFirst(executors []*WorkerSession, jobType string, policy schedulerPolicy) (*WorkerSession, func(), bool) {
|
||||
for _, executor := range executors {
|
||||
release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy)
|
||||
if ok {
|
||||
return executor, release, true
|
||||
}
|
||||
}
|
||||
return nil, nil, false
|
||||
}
|
||||
|
||||
func (r *Plugin) reserveExecutorByID(jobType string, policy schedulerPolicy, workerID string) (*WorkerSession, func(), error) {
|
||||
workerID = strings.TrimSpace(workerID)
|
||||
if workerID == "" {
|
||||
return nil, nil, fmt.Errorf("executor id is empty")
|
||||
}
|
||||
executors, err := r.registry.ListExecutors(jobType)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for _, executor := range executors {
|
||||
if executor == nil || executor.WorkerID != workerID {
|
||||
continue
|
||||
}
|
||||
release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy)
|
||||
if !ok {
|
||||
return nil, nil, errExecutorAtCapacity
|
||||
}
|
||||
return executor, release, nil
|
||||
}
|
||||
return nil, nil, fmt.Errorf("executor %s is not available for %s", workerID, jobType)
|
||||
}
|
||||
|
||||
func (r *Plugin) executorReservationLoad(executor *WorkerSession, lane SchedulerLane) int {
|
||||
if executor == nil {
|
||||
return 0
|
||||
}
|
||||
load := 0
|
||||
if executor.Heartbeat != nil && executor.Heartbeat.ExecutionSlotsUsed > 0 {
|
||||
load = int(executor.Heartbeat.ExecutionSlotsUsed)
|
||||
}
|
||||
workerID := strings.TrimSpace(executor.WorkerID)
|
||||
if ls := r.lanes[lane]; ls != nil {
|
||||
ls.execMu.Lock()
|
||||
load += ls.execRes[workerID]
|
||||
ls.execMu.Unlock()
|
||||
return load
|
||||
}
|
||||
r.schedulerExecMu.Lock()
|
||||
load += r.schedulerExecReservations[workerID]
|
||||
r.schedulerExecMu.Unlock()
|
||||
return load
|
||||
}
|
||||
|
||||
func sortExecutorsByLoad(executors []*WorkerSession, load func(*WorkerSession) int) {
|
||||
sort.SliceStable(executors, func(i, j int) bool {
|
||||
if executors[i] == nil {
|
||||
return false
|
||||
}
|
||||
if executors[j] == nil {
|
||||
return true
|
||||
}
|
||||
li, lj := load(executors[i]), load(executors[j])
|
||||
if li != lj {
|
||||
return li < lj
|
||||
}
|
||||
return executors[i].WorkerID < executors[j].WorkerID
|
||||
})
|
||||
}
|
||||
|
||||
// tryReserveExecutorCapacityForLane reserves an execution slot on the
|
||||
// per-lane reservation pool so that lanes cannot starve each other.
|
||||
func (r *Plugin) tryReserveExecutorCapacityForLane(
|
||||
@@ -1577,6 +1656,8 @@ func (r *Plugin) DispatchProposals(
|
||||
ExecutionTimeout: defaultScheduledExecutionTimeout,
|
||||
RetryBackoff: defaultScheduledRetryBackoff,
|
||||
ExecutorReserveBackoff: 200 * time.Millisecond,
|
||||
LocalQueuePin: defaultLocalQueuePin,
|
||||
OrphanRehomeGrace: defaultOrphanRehomeGrace,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -61,6 +61,12 @@ func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) {
|
||||
if policy.JobTypeMaxRuntime != 30*time.Minute {
|
||||
t.Fatalf("unexpected max runtime: got=%v", policy.JobTypeMaxRuntime)
|
||||
}
|
||||
if policy.LocalQueuePin != defaultLocalQueuePin {
|
||||
t.Fatalf("unexpected local queue pin: got=%d", policy.LocalQueuePin)
|
||||
}
|
||||
if policy.OrphanRehomeGrace != defaultOrphanRehomeGrace {
|
||||
t.Fatalf("unexpected orphan rehome grace: got=%v", policy.OrphanRehomeGrace)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T) {
|
||||
@@ -138,13 +144,13 @@ func TestReserveScheduledExecutorRespectsPerWorkerLimit(t *testing.T) {
|
||||
ExecutorReserveBackoff: time.Millisecond,
|
||||
}
|
||||
|
||||
executor1, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
|
||||
executor1, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve executor 1: %v", err)
|
||||
}
|
||||
defer release1()
|
||||
|
||||
executor2, release2, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
|
||||
executor2, release2, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve executor 2: %v", err)
|
||||
}
|
||||
@@ -266,7 +272,7 @@ func TestReserveScheduledExecutorTimesOutWhenNoExecutor(t *testing.T) {
|
||||
|
||||
start := time.Now()
|
||||
pluginSvc.Shutdown()
|
||||
_, _, err = pluginSvc.reserveScheduledExecutor(context.Background(), "missing-job-type", policy)
|
||||
_, _, err = pluginSvc.reserveScheduledExecutor(context.Background(), "missing-job-type", policy, nil)
|
||||
if err == nil {
|
||||
t.Fatalf("expected reservation shutdown error")
|
||||
}
|
||||
@@ -297,7 +303,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
|
||||
ExecutorReserveBackoff: 5 * time.Millisecond,
|
||||
}
|
||||
|
||||
_, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
|
||||
_, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve executor 1: %v", err)
|
||||
}
|
||||
@@ -308,7 +314,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
|
||||
}
|
||||
secondReserveCh := make(chan reserveResult, 1)
|
||||
go func() {
|
||||
_, release2, reserveErr := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
|
||||
_, release2, reserveErr := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
|
||||
if release2 != nil {
|
||||
release2()
|
||||
}
|
||||
@@ -992,3 +998,50 @@ func TestLaneStatus_LifecycleNextDetectionSurvivesDefaultLanePrune(t *testing.T)
|
||||
status.NextDetectionAt, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReserveWaterfillsPreferredHosts(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
ids := []string{"worker-a", "worker-b", "worker-c", "worker-d"}
|
||||
for _, id := range ids {
|
||||
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
|
||||
WorkerId: id,
|
||||
Address: "10.0.0.1:9334",
|
||||
Capabilities: []*plugin_pb.JobTypeCapability{
|
||||
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 8},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 8,
|
||||
ExecutorReserveBackoff: time.Millisecond,
|
||||
LocalQueuePin: 5,
|
||||
}
|
||||
preferred := []string{"10.0.0.1:8080"}
|
||||
|
||||
seen := map[string]int{}
|
||||
var releases []func()
|
||||
defer func() {
|
||||
for _, release := range releases {
|
||||
release()
|
||||
}
|
||||
}()
|
||||
for i := 0; i < 4; i++ {
|
||||
exec, release, err := pluginSvc.reserveScheduledExecutor(context.Background(), "erasure_coding", policy, preferred)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve %d: %v", i, err)
|
||||
}
|
||||
releases = append(releases, release)
|
||||
seen[exec.WorkerID]++
|
||||
}
|
||||
if len(seen) != 4 {
|
||||
t.Fatalf("want one job per worker, got %v", seen)
|
||||
}
|
||||
}
|
||||
@@ -277,14 +277,24 @@ func (r *Registry) PickDetector(jobType string) (*WorkerSession, error) {
|
||||
}
|
||||
|
||||
// PickExecutor picks one executor worker for a job type.
|
||||
func (r *Registry) PickExecutor(jobType string) (*WorkerSession, error) {
|
||||
return r.pickByKind(jobType, false)
|
||||
// preferredHosts, when set, ranks workers whose Hello Address host matches a
|
||||
// replica/source host ahead of others (same-machine EC pull). Empty means the
|
||||
// previous slots+round-robin order.
|
||||
func (r *Registry) PickExecutor(jobType string, preferredHosts ...string) (*WorkerSession, error) {
|
||||
list, err := r.ListExecutors(jobType, preferredHosts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return list[0], nil
|
||||
}
|
||||
|
||||
// ListExecutors returns sorted executor candidates for one job type.
|
||||
// Ordering is by most available execution slots, then lexical worker ID.
|
||||
// The top tie group is rotated round-robin to prevent sticky assignment.
|
||||
func (r *Registry) ListExecutors(jobType string) ([]*WorkerSession, error) {
|
||||
// preferredHosts, when any worker matches, are listed first (then the rest);
|
||||
// rotation applies only inside the preferred group so a local worker is not
|
||||
// rotated away in favor of a remote one.
|
||||
func (r *Registry) ListExecutors(jobType string, preferredHosts ...string) ([]*WorkerSession, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
@@ -293,8 +303,16 @@ func (r *Registry) ListExecutors(jobType string) ([]*WorkerSession, error) {
|
||||
return nil, fmt.Errorf("no executor worker available for job_type=%s", jobType)
|
||||
}
|
||||
|
||||
sortByKind(candidates, jobType, false)
|
||||
r.rotateTopCandidatesLocked(candidates, jobType, false)
|
||||
pref, rest := partitionByPreferredHost(candidates, preferredHosts)
|
||||
if len(pref) > 0 {
|
||||
sortByKind(pref, jobType, false)
|
||||
r.rotateTopCandidatesLocked(pref, jobType, false)
|
||||
sortByKind(rest, jobType, false)
|
||||
candidates = append(pref, rest...)
|
||||
} else {
|
||||
sortByKind(candidates, jobType, false)
|
||||
r.rotateTopCandidatesLocked(candidates, jobType, false)
|
||||
}
|
||||
|
||||
out := make([]*WorkerSession, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
|
||||
@@ -401,3 +401,70 @@ func TestRegistryHasCapableWorkerIgnoresNilCapability(t *testing.T) {
|
||||
t.Fatalf("expected HasCapableWorker to return false when capability has no detect or execute")
|
||||
}
|
||||
}
|
||||
|
||||
func TestListExecutorsPrefersMatchingReplicaHost(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
r := NewRegistry()
|
||||
r.UpsertFromHello(&plugin_pb.WorkerHello{
|
||||
WorkerId: "worker-remote",
|
||||
Address: "10.87.0.4:33646",
|
||||
Capabilities: []*plugin_pb.JobTypeCapability{
|
||||
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 8},
|
||||
},
|
||||
})
|
||||
r.UpsertFromHello(&plugin_pb.WorkerHello{
|
||||
WorkerId: "worker-local",
|
||||
Address: "10.87.0.3:33646",
|
||||
Capabilities: []*plugin_pb.JobTypeCapability{
|
||||
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 1},
|
||||
},
|
||||
})
|
||||
r.UpdateHeartbeat("worker-remote", &plugin_pb.WorkerHeartbeat{
|
||||
WorkerId: "worker-remote", ExecutionSlotsUsed: 0, ExecutionSlotsTotal: 8,
|
||||
})
|
||||
r.UpdateHeartbeat("worker-local", &plugin_pb.WorkerHeartbeat{
|
||||
WorkerId: "worker-local", ExecutionSlotsUsed: 0, ExecutionSlotsTotal: 1,
|
||||
})
|
||||
|
||||
// Without preference the emptier remote worker wins on slots.
|
||||
plain, err := r.ListExecutors("erasure_coding")
|
||||
if err != nil {
|
||||
t.Fatalf("ListExecutors: %v", err)
|
||||
}
|
||||
if plain[0].WorkerID != "worker-remote" {
|
||||
t.Fatalf("without preference want remote first, got %s", plain[0].WorkerID)
|
||||
}
|
||||
|
||||
// Replica on :8080 of the same host as worker-local.
|
||||
pref, err := r.ListExecutors("erasure_coding", "10.87.0.3:8080")
|
||||
if err != nil {
|
||||
t.Fatalf("ListExecutors preferred: %v", err)
|
||||
}
|
||||
if pref[0].WorkerID != "worker-local" {
|
||||
t.Fatalf("preferred host want worker-local first, got %s", pref[0].WorkerID)
|
||||
}
|
||||
if pref[1].WorkerID != "worker-remote" {
|
||||
t.Fatalf("remote should still be fallback, got %s", pref[1].WorkerID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobReplicaHostsFromSourcesAndReplicaServers(t *testing.T) {
|
||||
t.Parallel()
|
||||
job := &plugin_pb.JobSpec{
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"source_server": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "10.87.0.1:8080"},
|
||||
},
|
||||
"replica_servers": {
|
||||
Kind: &plugin_pb.ConfigValue_StringList{
|
||||
StringList: &plugin_pb.StringList{Values: []string{"10.87.0.1:8080", "10.87.0.2:8081"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
got := JobReplicaHosts(job)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
)
|
||||
|
||||
// queuedJob is one scheduled spec parked on a home worker's admin-side queue.
|
||||
type queuedJob struct {
|
||||
job *plugin_pb.JobSpec
|
||||
home string
|
||||
}
|
||||
|
||||
// workerJobQueues holds per-worker FIFOs for one dispatch batch.
|
||||
// Home assignment is done once up front; run goroutines only pop/steal.
|
||||
// leftover is the homeless FIFO: replica hostnames with no currently
|
||||
// registered worker. Orphan grace does not apply here.
|
||||
type workerJobQueues struct {
|
||||
mu sync.Mutex
|
||||
q map[string][]*queuedJob
|
||||
leftover []*queuedJob
|
||||
remaining int
|
||||
missingSince map[string]time.Time
|
||||
homes map[string]struct{}
|
||||
}
|
||||
|
||||
func newWorkerJobQueues() *workerJobQueues {
|
||||
return &workerJobQueues{
|
||||
q: make(map[string][]*queuedJob),
|
||||
missingSince: make(map[string]time.Time),
|
||||
homes: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) hasHome(workerID string) bool {
|
||||
if qs == nil || workerID == "" {
|
||||
return false
|
||||
}
|
||||
_, ok := qs.homes[workerID]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) push(home string, job *plugin_pb.JobSpec) {
|
||||
if qs == nil || job == nil {
|
||||
return
|
||||
}
|
||||
item := &queuedJob{job: job, home: home}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
qs.pushLocked(item, false)
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) pushFront(item *queuedJob) {
|
||||
if qs == nil || item == nil {
|
||||
return
|
||||
}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
qs.pushLocked(item, true)
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) pushLocked(item *queuedJob, front bool) {
|
||||
qs.remaining++
|
||||
if item.home == "" {
|
||||
if front {
|
||||
qs.leftover = append([]*queuedJob{item}, qs.leftover...)
|
||||
return
|
||||
}
|
||||
qs.leftover = append(qs.leftover, item)
|
||||
return
|
||||
}
|
||||
if front {
|
||||
qs.q[item.home] = append([]*queuedJob{item}, qs.q[item.home]...)
|
||||
return
|
||||
}
|
||||
qs.q[item.home] = append(qs.q[item.home], item)
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) isEmpty() bool {
|
||||
if qs == nil {
|
||||
return true
|
||||
}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
return qs.remaining == 0
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) queueLen(workerID string) int {
|
||||
if qs == nil {
|
||||
return 0
|
||||
}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
return len(qs.q[workerID])
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) leftoverLen() int {
|
||||
if qs == nil {
|
||||
return 0
|
||||
}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
return len(qs.leftover)
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) drain() []*queuedJob {
|
||||
if qs == nil {
|
||||
return nil
|
||||
}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
out := make([]*queuedJob, 0, qs.remaining)
|
||||
for id, qq := range qs.q {
|
||||
out = append(out, qq...)
|
||||
qs.q[id] = nil
|
||||
}
|
||||
out = append(out, qs.leftover...)
|
||||
qs.leftover = nil
|
||||
qs.remaining = 0
|
||||
return out
|
||||
}
|
||||
|
||||
func popFront(qq []*queuedJob) (*queuedJob, []*queuedJob) {
|
||||
item := qq[0]
|
||||
if len(qq) == 1 {
|
||||
return item, nil
|
||||
}
|
||||
return item, qq[1:]
|
||||
}
|
||||
|
||||
// assignJobsToWorkerQueues parks each job on the least-loaded replica-host
|
||||
// worker. A home exists only if a currently registered worker's -address
|
||||
// hostname matches a replica host. Otherwise the job is homeless: global
|
||||
// leftover FIFO, no fake home, no K, no orphan grace. Single-threaded.
|
||||
func (r *Plugin) assignJobsToWorkerQueues(
|
||||
jobType string,
|
||||
jobs []*plugin_pb.JobSpec,
|
||||
_ schedulerPolicy,
|
||||
) *workerJobQueues {
|
||||
qs := newWorkerJobQueues()
|
||||
executors, err := r.registry.ListExecutors(jobType)
|
||||
lane := JobTypeLane(jobType)
|
||||
queued := make(map[string]int, len(executors))
|
||||
|
||||
for _, job := range jobs {
|
||||
if job == nil {
|
||||
continue
|
||||
}
|
||||
if err != nil || len(executors) == 0 {
|
||||
qs.push("", job)
|
||||
continue
|
||||
}
|
||||
pref, _ := partitionByPreferredHost(executors, JobReplicaHosts(job))
|
||||
if len(pref) == 0 {
|
||||
// Never registered for that hostname: homeless, not an orphan.
|
||||
qs.push("", job)
|
||||
continue
|
||||
}
|
||||
for _, e := range pref {
|
||||
if e != nil && e.WorkerID != "" {
|
||||
qs.homes[e.WorkerID] = struct{}{}
|
||||
}
|
||||
}
|
||||
sortExecutorsByLoad(pref, func(e *WorkerSession) int {
|
||||
return r.executorReservationLoad(e, lane) + queued[e.WorkerID]
|
||||
})
|
||||
home := pref[0].WorkerID
|
||||
qs.push(home, job)
|
||||
queued[home]++
|
||||
r.bindQueuedJobWorker(job, home)
|
||||
}
|
||||
return qs
|
||||
}
|
||||
|
||||
// takeQueuedJob pops a job for a free worker.
|
||||
// Home worker (hostname matched a replica / has a home-queue identity):
|
||||
// 1. own home queue
|
||||
// 2. steal from another home if len>=K, or orphan past OrphanRehomeGrace
|
||||
// 3. homeless leftover FIFO
|
||||
// No-home worker (no matching replica, no home-queue identity):
|
||||
// 1. homeless leftover FIFO
|
||||
// 2. only if leftover empty: steal (K / orphan), same rules
|
||||
func (r *Plugin) takeQueuedJob(
|
||||
qs *workerJobQueues,
|
||||
jobType string,
|
||||
policy schedulerPolicy,
|
||||
) (item *queuedJob, targetID string) {
|
||||
if qs == nil {
|
||||
return nil, ""
|
||||
}
|
||||
executors, err := r.registry.ListExecutors(jobType)
|
||||
if err != nil || len(executors) == 0 {
|
||||
return nil, ""
|
||||
}
|
||||
lane := JobTypeLane(jobType)
|
||||
pin := policy.LocalQueuePin
|
||||
defer func() {
|
||||
if item != nil && targetID != "" {
|
||||
r.bindQueuedJobWorker(item.job, targetID)
|
||||
}
|
||||
}()
|
||||
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
if qs.remaining == 0 {
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
var bestHome *WorkerSession
|
||||
bestLoad := 0
|
||||
for _, exec := range executors {
|
||||
if len(qs.q[exec.WorkerID]) == 0 {
|
||||
continue
|
||||
}
|
||||
limit := schedulerWorkerExecutionLimit(exec, jobType, policy)
|
||||
if limit <= 0 {
|
||||
continue
|
||||
}
|
||||
load := r.executorReservationLoad(exec, lane)
|
||||
if load >= limit {
|
||||
continue
|
||||
}
|
||||
if bestHome == nil || load < bestLoad || (load == bestLoad && exec.WorkerID < bestHome.WorkerID) {
|
||||
bestHome = exec
|
||||
bestLoad = load
|
||||
}
|
||||
}
|
||||
if bestHome != nil {
|
||||
id := bestHome.WorkerID
|
||||
item, qs.q[id] = popFront(qs.q[id])
|
||||
qs.remaining--
|
||||
return item, id
|
||||
}
|
||||
|
||||
var thief *WorkerSession
|
||||
for _, exec := range executors {
|
||||
limit := schedulerWorkerExecutionLimit(exec, jobType, policy)
|
||||
if limit <= 0 {
|
||||
continue
|
||||
}
|
||||
if r.executorReservationLoad(exec, lane) >= limit {
|
||||
continue
|
||||
}
|
||||
if thief == nil || exec.WorkerID < thief.WorkerID {
|
||||
thief = exec
|
||||
}
|
||||
}
|
||||
if thief == nil {
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
takeLeftover := func() (*queuedJob, string) {
|
||||
if len(qs.leftover) == 0 {
|
||||
return nil, ""
|
||||
}
|
||||
item, qs.leftover = popFront(qs.leftover)
|
||||
qs.remaining--
|
||||
return item, thief.WorkerID
|
||||
}
|
||||
if !qs.hasHome(thief.WorkerID) {
|
||||
if item, id := takeLeftover(); item != nil {
|
||||
return item, id
|
||||
}
|
||||
}
|
||||
|
||||
live := make(map[string]struct{}, len(executors))
|
||||
for _, exec := range executors {
|
||||
live[exec.WorkerID] = struct{}{}
|
||||
delete(qs.missingSince, exec.WorkerID)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
grace := policy.OrphanRehomeGrace
|
||||
victimID := ""
|
||||
victimLen := 0
|
||||
victimOrphan := false
|
||||
for id, qq := range qs.q {
|
||||
if len(qq) == 0 {
|
||||
delete(qs.missingSince, id)
|
||||
continue
|
||||
}
|
||||
_, isLive := live[id]
|
||||
orphan := !isLive
|
||||
if isLive {
|
||||
if pin > 0 && len(qq) < pin {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
if _, ok := qs.missingSince[id]; !ok {
|
||||
qs.missingSince[id] = now
|
||||
}
|
||||
if grace > 0 && now.Sub(qs.missingSince[id]) < grace {
|
||||
continue
|
||||
}
|
||||
}
|
||||
better := victimID == ""
|
||||
if !better && orphan != victimOrphan {
|
||||
better = orphan
|
||||
}
|
||||
if !better && orphan == victimOrphan {
|
||||
better = len(qq) > victimLen || (len(qq) == victimLen && id < victimID)
|
||||
}
|
||||
if better {
|
||||
victimID = id
|
||||
victimLen = len(qq)
|
||||
victimOrphan = orphan
|
||||
}
|
||||
}
|
||||
if victimID != "" {
|
||||
item, qs.q[victimID] = popFront(qs.q[victimID])
|
||||
qs.remaining--
|
||||
return item, thief.WorkerID
|
||||
}
|
||||
return takeLeftover()
|
||||
}
|
||||
@@ -0,0 +1,600 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
)
|
||||
|
||||
func replicaJob(id, source string) *plugin_pb.JobSpec {
|
||||
return &plugin_pb.JobSpec{
|
||||
JobId: id,
|
||||
JobType: "erasure_coding",
|
||||
Summary: id,
|
||||
DedupeKey: id,
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"source_server": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: source},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func registerECWorkers(t *testing.T, pluginSvc *Plugin, workers map[string]string) {
|
||||
t.Helper()
|
||||
for id, addr := range workers {
|
||||
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
|
||||
WorkerId: id,
|
||||
Address: addr,
|
||||
Capabilities: []*plugin_pb.JobTypeCapability{
|
||||
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 8},
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignJobsWaterfillsPreferredWorkers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-a": "10.0.0.1:9334",
|
||||
"worker-b": "10.0.0.2:9334",
|
||||
"worker-c": "10.0.0.3:9334",
|
||||
"worker-d": "10.0.0.4:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("j1", "10.0.0.1:8080"),
|
||||
replicaJob("j2", "10.0.0.1:8080"),
|
||||
replicaJob("j3", "10.0.0.1:8080"),
|
||||
replicaJob("j4", "10.0.0.1:8080"),
|
||||
}
|
||||
// All four replica IPs listed so every worker is preferred.
|
||||
for _, job := range jobs {
|
||||
job.Parameters["replica_servers"] = &plugin_pb.ConfigValue{
|
||||
Kind: &plugin_pb.ConfigValue_StringList{StringList: &plugin_pb.StringList{
|
||||
Values: []string{"10.0.0.1:8080", "10.0.0.2:8080", "10.0.0.3:8080", "10.0.0.4:8080"},
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, schedulerPolicy{LocalQueuePin: 5})
|
||||
if qs.queueLen("worker-a")+qs.queueLen("worker-b")+qs.queueLen("worker-c")+qs.queueLen("worker-d") != 4 {
|
||||
t.Fatalf("expected 4 assigned jobs")
|
||||
}
|
||||
for _, id := range []string{"worker-a", "worker-b", "worker-c", "worker-d"} {
|
||||
if qs.queueLen(id) != 1 {
|
||||
t.Fatalf("want 1 job on %s, got %d", id, qs.queueLen(id))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignAndTakeDoesNotStarveOtherHomes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("local-1", "10.0.0.1:8080"),
|
||||
replicaJob("local-2", "10.0.0.1:8080"),
|
||||
replicaJob("remote-1", "10.0.0.2:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 2 || qs.queueLen("worker-remote") != 1 {
|
||||
t.Fatalf("homes local=%d remote=%d", qs.queueLen("worker-local"), qs.queueLen("worker-remote"))
|
||||
}
|
||||
|
||||
_, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
_, release1, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, target1)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve 1: %v", err)
|
||||
}
|
||||
defer release1()
|
||||
|
||||
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item2 == nil || target2 != "worker-remote" {
|
||||
t.Fatalf("second take should start remote home, got job=%v target=%s", item2, target2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakeStealsOnlyWhenHomeQueueReachesK(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := make([]*plugin_pb.JobSpec, 0, 6)
|
||||
for i := 0; i < 6; i++ {
|
||||
jobs = append(jobs, replicaJob(fmt.Sprintf("j%d", i), "10.0.0.1:8080"))
|
||||
}
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 6 {
|
||||
t.Fatalf("all jobs should home on local, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
_, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if target != "worker-local" {
|
||||
t.Fatalf("first take want local, got %s", target)
|
||||
}
|
||||
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
|
||||
if err != nil {
|
||||
t.Fatalf("reserve local: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
// 5 still queued on local (>=K) and local has no slot → steal to remote.
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-remote" {
|
||||
t.Fatalf("expected steal to remote, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
if qs.queueLen("worker-local") != 4 {
|
||||
t.Fatalf("after steal want 4 left on local, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
// 4 < K: do not steal again.
|
||||
if item, tgt := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy); item != nil {
|
||||
t.Fatalf("pin should hold remaining jobs, got target=%s", tgt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakeStealsImmediatelyWhenPinDisabled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("a", "10.0.0.1:8080"),
|
||||
replicaJob("b", "10.0.0.1:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 0}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
|
||||
_, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if target != "worker-local" {
|
||||
t.Fatalf("first take want local, got %s", target)
|
||||
}
|
||||
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
|
||||
if err != nil {
|
||||
t.Fatalf("reserve local: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-remote" {
|
||||
t.Fatalf("K=0 should steal immediately, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakeStealsOrphanedHomeWithoutK(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("o1", "10.0.0.1:8080"),
|
||||
replicaJob("o2", "10.0.0.1:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 1,
|
||||
LocalQueuePin: 5,
|
||||
OrphanRehomeGrace: 30 * time.Second,
|
||||
}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 2 {
|
||||
t.Fatalf("want 2 on local, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
pluginSvc.registry.Remove("worker-local")
|
||||
|
||||
if item, tgt := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy); item != nil {
|
||||
t.Fatalf("within grace should hold orphan queue, got target=%s", tgt)
|
||||
}
|
||||
if qs.queueLen("worker-local") != 2 {
|
||||
t.Fatalf("held queue changed: %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
qs.mu.Lock()
|
||||
qs.missingSince["worker-local"] = time.Now().Add(-time.Minute)
|
||||
qs.mu.Unlock()
|
||||
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-remote" {
|
||||
t.Fatalf("after grace should steal without K, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
if qs.queueLen("worker-local") != 1 {
|
||||
t.Fatalf("want 1 orphan left, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
stolen2, stealTarget2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen2 == nil || stealTarget2 != "worker-remote" {
|
||||
t.Fatalf("second orphan should also move, got job=%v target=%s", stolen2, stealTarget2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakeKeepsOrphanWhenHomeReconnectsDuringGrace(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{replicaJob("blink", "10.0.0.1:8080")}
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 1,
|
||||
LocalQueuePin: 5,
|
||||
OrphanRehomeGrace: 30 * time.Second,
|
||||
}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
pluginSvc.registry.Remove("worker-local")
|
||||
if item, _ := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy); item != nil {
|
||||
t.Fatal("expected hold during grace")
|
||||
}
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{"worker-local": "10.0.0.1:9334"})
|
||||
item, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item == nil || target != "worker-local" {
|
||||
t.Fatalf("reconnected home should run its queue, got job=%v target=%s", item, target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakeStealsOrphanImmediatelyWhenGraceDisabled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{replicaJob("o1", "10.0.0.1:8080")}
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5, OrphanRehomeGrace: 0}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
pluginSvc.registry.Remove("worker-local")
|
||||
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-remote" {
|
||||
t.Fatalf("grace=0 should steal immediately, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBindQueuedJobWorkerKeepsPendingState(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
job := replicaJob("bind-1", "10.0.0.1:8080")
|
||||
pluginSvc.trackExecutionQueued(job)
|
||||
pluginSvc.bindQueuedJobWorker(job, "worker-local")
|
||||
|
||||
pluginSvc.jobsMu.Lock()
|
||||
tracked := pluginSvc.jobs[job.JobId]
|
||||
pluginSvc.jobsMu.Unlock()
|
||||
if tracked == nil {
|
||||
t.Fatal("missing tracked job")
|
||||
}
|
||||
if tracked.State != "job_state_pending" {
|
||||
t.Fatalf("state=%s", tracked.State)
|
||||
}
|
||||
if tracked.Stage != "queued" {
|
||||
t.Fatalf("stage=%s", tracked.Stage)
|
||||
}
|
||||
if tracked.WorkerID != "worker-local" {
|
||||
t.Fatalf("worker=%s", tracked.WorkerID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomelessAllRemoteWorkersUseFIFOWithoutPinOrGrace(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
// Workers never registered on the replica hostname — homeless, not orphan.
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-a": "10.0.0.5:9334",
|
||||
"worker-b": "10.0.0.6:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("h1", "10.0.0.1:8080"),
|
||||
replicaJob("h2", "10.0.0.1:8080"),
|
||||
replicaJob("h3", "10.0.0.1:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 1,
|
||||
LocalQueuePin: 5,
|
||||
OrphanRehomeGrace: 30 * time.Second,
|
||||
}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-a") != 0 || qs.queueLen("worker-b") != 0 {
|
||||
t.Fatalf("homeless jobs must not get a fake home, a=%d b=%d", qs.queueLen("worker-a"), qs.queueLen("worker-b"))
|
||||
}
|
||||
if qs.leftoverLen() != 3 {
|
||||
t.Fatalf("want 3 homeless FIFO, got %d", qs.leftoverLen())
|
||||
}
|
||||
|
||||
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item1 == nil || item1.job.JobId != "h1" {
|
||||
t.Fatalf("FIFO first want h1, got %v", item1)
|
||||
}
|
||||
_, release1, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, target1)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve 1: %v", err)
|
||||
}
|
||||
defer release1()
|
||||
|
||||
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item2 == nil || item2.job.JobId != "h2" {
|
||||
t.Fatalf("FIFO second want h2 immediately, got %v", item2)
|
||||
}
|
||||
if target2 == "" || target2 == target1 {
|
||||
t.Fatalf("second take should hit the other idle worker, t1=%s t2=%s", target1, target2)
|
||||
}
|
||||
if qs.leftoverLen() != 1 {
|
||||
t.Fatalf("want 1 homeless left, got %d", qs.leftoverLen())
|
||||
}
|
||||
}
|
||||
|
||||
func TestMixedReplicaLocalJobsStayHomeBeforeHomeless(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("local-1", "10.0.0.1:8080"),
|
||||
replicaJob("local-2", "10.0.0.1:8080"),
|
||||
replicaJob("homeless-1", "10.0.0.9:8080"),
|
||||
replicaJob("homeless-2", "10.0.0.9:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 1,
|
||||
LocalQueuePin: 5,
|
||||
OrphanRehomeGrace: 30 * time.Second,
|
||||
}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 2 || qs.queueLen("worker-remote") != 0 || qs.leftoverLen() != 2 {
|
||||
t.Fatalf("homes local=%d remote=%d leftover=%d", qs.queueLen("worker-local"), qs.queueLen("worker-remote"), qs.leftoverLen())
|
||||
}
|
||||
|
||||
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item1 == nil || item1.job.JobId != "local-1" || target1 != "worker-local" {
|
||||
t.Fatalf("first take want local-1 on home, got job=%v target=%s", item1, target1)
|
||||
}
|
||||
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
|
||||
if err != nil {
|
||||
t.Fatalf("reserve local: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item2 == nil || item2.job.JobId != "homeless-1" || target2 != "worker-remote" {
|
||||
t.Fatalf("remote should take homeless, not steal pinned home, got job=%v target=%s", item2, target2)
|
||||
}
|
||||
if qs.queueLen("worker-local") != 1 {
|
||||
t.Fatalf("remaining replica-local job must stay home, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
if qs.leftoverLen() != 1 {
|
||||
t.Fatalf("one homeless should remain, leftover=%d", qs.leftoverLen())
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdleHomeWorkerTakesHomelessAfterEmptyHomeQueue(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("home-1", "10.0.0.1:8080"),
|
||||
replicaJob("homeless-1", "10.0.0.9:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 1,
|
||||
LocalQueuePin: 5,
|
||||
OrphanRehomeGrace: 30 * time.Second,
|
||||
}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 1 || qs.leftoverLen() != 1 {
|
||||
t.Fatalf("home=%d leftover=%d", qs.queueLen("worker-local"), qs.leftoverLen())
|
||||
}
|
||||
|
||||
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item1 == nil || item1.job.JobId != "home-1" || target1 != "worker-local" {
|
||||
t.Fatalf("first take want home-1, got job=%v target=%s", item1, target1)
|
||||
}
|
||||
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item2 == nil || item2.job.JobId != "homeless-1" || target2 != "worker-local" {
|
||||
t.Fatalf("idle home with empty queue may take homeless, got job=%v target=%s", item2, target2)
|
||||
}
|
||||
if qs.leftoverLen() != 0 {
|
||||
t.Fatalf("homeless should be gone, leftover=%d", qs.leftoverLen())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomelessWorkerTakesFIFOBeforeStealableHome(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := make([]*plugin_pb.JobSpec, 0, 7)
|
||||
for i := 0; i < 6; i++ {
|
||||
jobs = append(jobs, replicaJob(fmt.Sprintf("j%d", i), "10.0.0.1:8080"))
|
||||
}
|
||||
jobs = append(jobs, replicaJob("homeless-1", "10.0.0.9:8080"))
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 6 || qs.leftoverLen() != 1 {
|
||||
t.Fatalf("home=%d leftover=%d", qs.queueLen("worker-local"), qs.leftoverLen())
|
||||
}
|
||||
if qs.hasHome("worker-remote") {
|
||||
t.Fatal("remote must have no home identity")
|
||||
}
|
||||
|
||||
_, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if target != "worker-local" {
|
||||
t.Fatalf("first take want local, got %s", target)
|
||||
}
|
||||
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
|
||||
if err != nil {
|
||||
t.Fatalf("reserve local: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
item, takeTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item == nil || takeTarget != "worker-remote" || item.job.JobId != "homeless-1" {
|
||||
t.Fatalf("remote must take homeless FIFO before K-steal, got job=%v target=%s", item, takeTarget)
|
||||
}
|
||||
if qs.leftoverLen() != 0 {
|
||||
t.Fatalf("homeless should be gone, leftover=%d", qs.leftoverLen())
|
||||
}
|
||||
if qs.queueLen("worker-local") != 5 {
|
||||
t.Fatalf("stealable home must stay until FIFO empty, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-remote" || stolen.job.JobId == "homeless-1" {
|
||||
t.Fatalf("after FIFO empty remote may steal >=K, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomeWorkerStealsBeforeHomelessAfterOwnQueueEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-other": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := make([]*plugin_pb.JobSpec, 0, 8)
|
||||
for i := 0; i < 6; i++ {
|
||||
jobs = append(jobs, replicaJob(fmt.Sprintf("j%d", i), "10.0.0.1:8080"))
|
||||
}
|
||||
jobs = append(jobs, replicaJob("other-1", "10.0.0.2:8080"))
|
||||
jobs = append(jobs, replicaJob("homeless-1", "10.0.0.9:8080"))
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 6 || qs.queueLen("worker-other") != 1 || qs.leftoverLen() != 1 {
|
||||
t.Fatalf("local=%d other=%d leftover=%d", qs.queueLen("worker-local"), qs.queueLen("worker-other"), qs.leftoverLen())
|
||||
}
|
||||
if !qs.hasHome("worker-other") {
|
||||
t.Fatal("other must keep home identity")
|
||||
}
|
||||
|
||||
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item1 == nil || target1 != "worker-local" {
|
||||
t.Fatalf("first take want local home, got job=%v target=%s", item1, target1)
|
||||
}
|
||||
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
|
||||
if err != nil {
|
||||
t.Fatalf("reserve local: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item2 == nil || item2.job.JobId != "other-1" || target2 != "worker-other" {
|
||||
t.Fatalf("other should drain own home first, got job=%v target=%s", item2, target2)
|
||||
}
|
||||
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-other" || stolen.job.JobId == "homeless-1" {
|
||||
t.Fatalf("home worker with empty own queue still steals before homeless, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
if qs.leftoverLen() != 1 {
|
||||
t.Fatalf("homeless should still wait, leftover=%d", qs.leftoverLen())
|
||||
}
|
||||
}
|
||||
@@ -82,7 +82,7 @@ func runPluginWorkerWithOptions(options pluginWorkerRunOptions) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
handlers, err := buildPluginWorkerHandlers(options.JobTypes, dialOption, options.MaxExecute, options.WorkingDir)
|
||||
handlers, err := buildPluginWorkerHandlers(options.JobTypes, dialOption, options.MaxExecute, options.WorkingDir, options.Address)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to build plugin worker handlers: %v", err)
|
||||
return false
|
||||
@@ -141,7 +141,7 @@ func resolvePluginWorkerID(explicitID string, workingDir string) (string, error)
|
||||
// buildPluginWorkerHandlers resolves the comma-separated jobTypes string
|
||||
// (which may contain category names like "all", "default", "heavy" and/or
|
||||
// explicit job type names/aliases) into a deduplicated slice of JobHandlers.
|
||||
func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxExecute int, workingDir string) ([]pluginworker.JobHandler, error) {
|
||||
func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxExecute int, workingDir string, workerAddress ...string) ([]pluginworker.JobHandler, error) {
|
||||
jobTypes = strings.TrimSpace(jobTypes)
|
||||
if jobTypes == "" {
|
||||
jobTypes = defaultPluginWorkerJobTypes
|
||||
@@ -152,10 +152,16 @@ func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxE
|
||||
return nil, err
|
||||
}
|
||||
|
||||
addr := ""
|
||||
if len(workerAddress) > 0 {
|
||||
addr = strings.TrimSpace(workerAddress[0])
|
||||
}
|
||||
|
||||
opts := pluginworker.HandlerBuildOptions{
|
||||
GrpcDialOption: dialOption,
|
||||
MaxExecute: maxExecute,
|
||||
WorkingDir: workingDir,
|
||||
WorkerAddress: addr,
|
||||
}
|
||||
|
||||
handlers := make([]pluginworker.JobHandler, 0, len(factories))
|
||||
|
||||
@@ -37,6 +37,9 @@ type HandlerBuildOptions struct {
|
||||
GrpcDialOption grpc.DialOption
|
||||
MaxExecute int
|
||||
WorkingDir string
|
||||
// WorkerAddress is the advertised weed worker -address (host:port).
|
||||
// It is not a volume-server URL; handlers match replicas by hostname.
|
||||
WorkerAddress string
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// HostKey is the hostname/IP of an "host:port" address (or the string itself
|
||||
// when it has no port). Compared case-insensitively so worker Hello
|
||||
// "10.87.0.3:33646" matches a volume replica on "10.87.0.3:8080".
|
||||
func HostKey(addr string) string {
|
||||
addr = strings.TrimSpace(addr)
|
||||
if addr == "" {
|
||||
return ""
|
||||
}
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return strings.ToLower(addr)
|
||||
}
|
||||
return strings.ToLower(host)
|
||||
}
|
||||
|
||||
// HostKeysFromAddresses returns unique HostKey values, skipping empties.
|
||||
func HostKeysFromAddresses(addrs []string) []string {
|
||||
seen := make(map[string]struct{}, len(addrs))
|
||||
var out []string
|
||||
for _, a := range addrs {
|
||||
h := HostKey(a)
|
||||
if h == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[h]; ok {
|
||||
continue
|
||||
}
|
||||
seen[h] = struct{}{}
|
||||
out = append(out, h)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// AddressIsLocal reports whether addr's host is this process's machine
|
||||
// (loopback or a configured interface address). Used so an EC worker prefers
|
||||
// a replica on the same box after replica sync.
|
||||
func AddressIsLocal(addr string) bool {
|
||||
host := HostKey(addr)
|
||||
if host == "" {
|
||||
return false
|
||||
}
|
||||
if host == "localhost" || host == "127.0.0.1" || host == "::1" {
|
||||
return true
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
if ip != nil && ip.IsLoopback() {
|
||||
return true
|
||||
}
|
||||
ifaces, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, ia := range ifaces {
|
||||
var got net.IP
|
||||
switch v := ia.(type) {
|
||||
case *net.IPNet:
|
||||
got = v.IP
|
||||
case *net.IPAddr:
|
||||
got = v.IP
|
||||
}
|
||||
if got == nil {
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(got.String(), host) {
|
||||
return true
|
||||
}
|
||||
if ip != nil && got.Equal(ip) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package util
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestHostKeyStripsPort(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := HostKey("10.87.0.3:8080"); got != "10.87.0.3" {
|
||||
t.Fatalf("HostKey replica: got %q", got)
|
||||
}
|
||||
if got := HostKey("10.87.0.3:33646"); got != "10.87.0.3" {
|
||||
t.Fatalf("HostKey worker: got %q", got)
|
||||
}
|
||||
if HostKey("10.87.0.3:8080") != HostKey("10.87.0.3:33646") {
|
||||
t.Fatal("worker and volume server on the same IP must share a HostKey")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostKeysFromAddressesDedups(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := HostKeysFromAddresses([]string{"10.0.0.1:8080", "10.0.0.1:8081", "10.0.0.2:8080", ""})
|
||||
if len(got) != 2 || got[0] != "10.0.0.1" || got[1] != "10.0.0.2" {
|
||||
t.Fatalf("got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
storagetypes "github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/volume_replica"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types/base"
|
||||
@@ -36,6 +37,9 @@ type ErasureCodingTask struct {
|
||||
workDir string
|
||||
progress float64
|
||||
grpcDialOption grpc.DialOption
|
||||
// workerAddress is weed worker -address (host:port). Not a volume URL;
|
||||
// late-bind matches replica hostnames against HostKey(workerAddress).
|
||||
workerAddress string
|
||||
|
||||
// EC parameters
|
||||
dataShards int32
|
||||
@@ -403,9 +407,75 @@ func (t *ErasureCodingTask) syncAndSelectSourceReplica() error {
|
||||
glog.Infof("EC encode: using best replica %s as source for volume %d (was %s)", best.Url, t.volumeID, t.server)
|
||||
t.server = best.Url
|
||||
}
|
||||
// Late-bind from after consistency: worker -address hostname, not Sources[0]
|
||||
// and not AddressIsLocal (replica URLs are hostnames like full-02:8081).
|
||||
workerAddr := strings.TrimSpace(t.workerAddress)
|
||||
if workerAddr == "" {
|
||||
workerAddr, _ = os.Hostname()
|
||||
}
|
||||
if local := preferWorkerHostReplicaIfComplete(t.grpcDialOption, needle.VolumeId(t.volumeID), locs, t.server, workerAddr); local != "" && local != t.server {
|
||||
glog.Infof("EC encode: preferring worker-host replica %s as source for volume %d (was %s)", local, t.volumeID, t.server)
|
||||
t.server = local
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type replicaFileCount struct {
|
||||
url string
|
||||
fileCount uint64
|
||||
err error
|
||||
}
|
||||
|
||||
// preferWorkerHostReplicaIfComplete rebinds from to a same-hostname replica
|
||||
// only when live FileCounts are equal. Diverged counts keep current (max).
|
||||
func preferWorkerHostReplicaIfComplete(dial grpc.DialOption, vid needle.VolumeId, locs []wdclient.Location, current, workerAddress string) string {
|
||||
statuses := volume_replica.GetReplicaStatuses(dial, vid, locs)
|
||||
replicas := make([]replicaFileCount, 0, len(statuses))
|
||||
for _, s := range statuses {
|
||||
replicas = append(replicas, replicaFileCount{url: s.Location.Url, fileCount: s.FileCount, err: s.Error})
|
||||
}
|
||||
return lateBindSourceToWorkerHost(current, workerAddress, replicas)
|
||||
}
|
||||
|
||||
// lateBindSourceToWorkerHost: equal FileCount → replica on worker -address host
|
||||
// (any port). Diverge → keep current (already max FileCount). Worker address
|
||||
// is not a volume URL.
|
||||
func lateBindSourceToWorkerHost(current, workerAddress string, replicas []replicaFileCount) string {
|
||||
workerHost := util.HostKey(workerAddress)
|
||||
if workerHost == "" {
|
||||
return current
|
||||
}
|
||||
var live []replicaFileCount
|
||||
for _, r := range replicas {
|
||||
if r.err != nil || r.url == "" {
|
||||
continue
|
||||
}
|
||||
live = append(live, r)
|
||||
}
|
||||
if len(live) == 0 {
|
||||
return current
|
||||
}
|
||||
maxCount := live[0].fileCount
|
||||
allSame := true
|
||||
for _, r := range live[1:] {
|
||||
if r.fileCount != live[0].fileCount {
|
||||
allSame = false
|
||||
}
|
||||
if r.fileCount > maxCount {
|
||||
maxCount = r.fileCount
|
||||
}
|
||||
}
|
||||
if !allSame {
|
||||
return current
|
||||
}
|
||||
for _, r := range live {
|
||||
if util.HostKey(r.url) == workerHost {
|
||||
return r.url
|
||||
}
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
// rollbackReadonly is a best-effort restore of every replica markReplicasReadonly
|
||||
// touched, used when the EC task fails before the originals are deleted. Logs but
|
||||
// does not return errors; uses a fresh context since the caller's may be cancelled.
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package erasure_coding
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestLateBindSourceToWorkerHostEqualFileCountPrefersWorkerHost(t *testing.T) {
|
||||
a := replicaFileCount{url: "full-01:8080", fileCount: 21}
|
||||
b := replicaFileCount{url: "full-02:8081", fileCount: 21}
|
||||
worker := "full-02:9334"
|
||||
|
||||
// Sources [A,B] — topology-first is A; worker on B must still copy B.
|
||||
if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{a, b}); got != b.url {
|
||||
t.Fatalf("Sources [A,B] worker B: got %s want %s", got, b.url)
|
||||
}
|
||||
// Invert Sources — still B.
|
||||
if got := lateBindSourceToWorkerHost(b.url, worker, []replicaFileCount{b, a}); got != b.url {
|
||||
t.Fatalf("Sources [B,A] worker B: got %s want %s", got, b.url)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLateBindSourceToWorkerHostDivergeKeepsMaxFileCount(t *testing.T) {
|
||||
a := replicaFileCount{url: "full-01:8080", fileCount: 40}
|
||||
b := replicaFileCount{url: "full-02:8081", fileCount: 21}
|
||||
worker := "full-02:9334"
|
||||
|
||||
if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{a, b}); got != a.url {
|
||||
t.Fatalf("diverge [A,B]: got %s want max FileCount %s", got, a.url)
|
||||
}
|
||||
if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{b, a}); got != a.url {
|
||||
t.Fatalf("diverge [B,A]: got %s want max FileCount %s", got, a.url)
|
||||
}
|
||||
}
|
||||
@@ -26,7 +26,9 @@ func init() {
|
||||
Category: pluginworker.CategoryHeavy,
|
||||
Aliases: []string{"erasure-coding", "erasure.coding", "ec"},
|
||||
Build: func(opts pluginworker.HandlerBuildOptions) (pluginworker.JobHandler, error) {
|
||||
return NewErasureCodingHandler(opts.GrpcDialOption, opts.WorkingDir), nil
|
||||
h := NewErasureCodingHandler(opts.GrpcDialOption, opts.WorkingDir)
|
||||
h.workerAddress = strings.TrimSpace(opts.WorkerAddress)
|
||||
return h, nil
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -39,6 +41,7 @@ type erasureCodingWorkerConfig struct {
|
||||
type ErasureCodingHandler struct {
|
||||
grpcDialOption grpc.DialOption
|
||||
workingDir string
|
||||
workerAddress string
|
||||
}
|
||||
|
||||
func NewErasureCodingHandler(grpcDialOption grpc.DialOption, workingDir string) *ErasureCodingHandler {
|
||||
@@ -519,6 +522,7 @@ func (h *ErasureCodingHandler) Execute(
|
||||
params.Collection,
|
||||
h.grpcDialOption,
|
||||
)
|
||||
task.workerAddress = h.workerAddress
|
||||
execCtx, execCancel := context.WithCancel(ctx)
|
||||
defer execCancel()
|
||||
task.SetProgressCallback(func(progress float64, stage string) {
|
||||
@@ -670,8 +674,15 @@ func buildErasureCodingProposal(
|
||||
}
|
||||
|
||||
sourceNode := ""
|
||||
if len(params.Sources) > 0 {
|
||||
sourceNode = strings.TrimSpace(params.Sources[0].Node)
|
||||
var replicaServers []string
|
||||
for _, s := range params.Sources {
|
||||
if s == nil || strings.TrimSpace(s.Node) == "" || len(s.ShardIds) > 0 {
|
||||
continue
|
||||
}
|
||||
replicaServers = append(replicaServers, s.Node)
|
||||
if sourceNode == "" {
|
||||
sourceNode = strings.TrimSpace(s.Node)
|
||||
}
|
||||
}
|
||||
|
||||
summary := fmt.Sprintf("Erasure code volume %d", result.VolumeID)
|
||||
@@ -702,6 +713,9 @@ func buildErasureCodingProposal(
|
||||
"source_server": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: sourceNode},
|
||||
},
|
||||
"replica_servers": {
|
||||
Kind: &plugin_pb.ConfigValue_StringList{StringList: &plugin_pb.StringList{Values: replicaServers}},
|
||||
},
|
||||
"collection": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: result.Collection},
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user