EC worker: pin jobs to replica-host workers, late-bind local source (#11718)

* feat: prefer same-host worker and local EC replica

Admin executor pick ranked idle workers only, so a job whose volume
lived on full-03 could still run on full-01 and pull the whole .dat
across the LAN. Rank workers whose Hello host matches any replica
first (fallback to others if the local worker is full).

Detection named Sources[0] / the lowest server id; execute then
SyncAndSelectBestReplica and could switch the copy source to another
rack. After sync, if a replica on this machine has a complete file
count, copy from that URL so affinity and pull stay on one host.

* feat: pin scheduler jobs to replica-host workers until load K

Reserve prefers the least-loaded replica-host worker and waits while
that load is below 5 instead of spilling to a free remote worker.
K=0 restores immediate spill. Waterfill uses reservation load so four
idle preferred workers each get a job.

* feat: park scheduled jobs on per-worker admin queues

Assign a detection batch to the least-loaded replica-host worker in
one pass, then let ExecutionConcurrency goroutines pop a home-runnable
job or steal when that home queue reaches K. Dispatch no longer holds
a job in reserve-backoff, so a free worker is not blocked by another
home's pin. Tracked jobs stay pending/queued and record the home
worker for the admin UI.

* fix: rehome queued jobs when the home worker dies

A live home still pins below K. A home missing from live executors
is stolen immediately onto a free worker so a short orphaned queue
does not sit until the dispatch window ends.

* feat: wait before stealing a dead home worker queue

Orphaned queues are held for 30s so a worker reconnect blink does
not dump jobs onto other machines. After the grace, steal without K
as before. Zero disables the wait. A home that comes back during
grace keeps its queue.

* fix: late-bind EC from to worker hostname

After replicas are consistent (equal FileCount), rebind encode source
to a volume server on the same hostname as worker -address. Any port
on that host is local. Diverged FileCount still uses the max replica.

* feat: assign homeless jobs to global FIFO

Jobs whose replica hostname has no registered worker skip pin and orphan grace.
Free workers take own home, then steal at K/orphan, then leftover FIFO.

* fix: take homeless FIFO before steal for no-home workers

Workers without a replica-host match take leftover first; steal (K/orphan)
only when FIFO is empty. Home workers keep own-queue, steal, then leftover.

---------

Co-authored-by: Chepurnov Simon <195125934+CHEPURNOV-S-S@users.noreply.github.com>
This commit is contained in:
Чепурнов СемёнandChepurnov Simon authored and GitHub committed 2026-10-13 05:31:34 +08:00
1 parent 492e700d1f
commit 624a13ecc6
16 files changed
+1601 -141

No files matched your search

+66
View File
@@ -0,0 +1,66 @@
package plugin
import (
"strings"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
)
// JobReplicaHosts are the volume-replica hosts a job will read from.
// Used so executor pick prefers a worker on one of those machines.
// Sources with ShardIds are EC-cleanup rows, not encode replicas.
func JobReplicaHosts(job *plugin_pb.JobSpec) []string {
if job == nil {
return nil
}
var addrs []string
addrs = append(addrs, pluginworker.ReadStringListConfig(job.Parameters, "replica_servers")...)
if src := strings.TrimSpace(pluginworker.ReadStringConfig(job.Parameters, "source_server", "")); src != "" {
addrs = append(addrs, src)
}
if payload := pluginworker.ReadBytesConfig(job.Parameters, "task_params_pb"); len(payload) > 0 {
params := &worker_pb.TaskParams{}
if err := proto.Unmarshal(payload, params); err == nil {
for _, s := range params.GetSources() {
if s == nil || s.Node == "" || len(s.ShardIds) > 0 {
continue
}
addrs = append(addrs, s.Node)
}
}
}
return util.HostKeysFromAddresses(addrs)
}
// partitionByPreferredHost splits candidates into (host matches, others).
// No preferred hosts, or no matches, returns (nil, all) so callers keep
// the existing slots+round-robin path.
func partitionByPreferredHost(candidates []*WorkerSession, preferredHosts []string) (pref, rest []*WorkerSession) {
want := util.HostKeysFromAddresses(preferredHosts)
if len(want) == 0 || len(candidates) == 0 {
return nil, candidates
}
set := make(map[string]struct{}, len(want))
for _, h := range want {
set[h] = struct{}{}
}
for _, c := range candidates {
if c == nil {
continue
}
if _, ok := set[util.HostKey(c.Address)]; ok {
pref = append(pref, c)
continue
}
rest = append(rest, c)
}
if len(pref) == 0 {
return nil, candidates
}
return pref, rest
}
+1 -1
View File
@@ -615,7 +615,7 @@ func (r *Plugin) ExecuteJob(
releaseGate := r.acquireDetectionExecutionGate(jobType, true)
defer releaseGate()
executor, err := r.registry.PickExecutor(jobType)
executor, err := r.registry.PickExecutor(jobType, JobReplicaHosts(job)...)
if err != nil {
return nil, err
}
+31
View File
@@ -887,6 +887,37 @@ func (r *Plugin) trackExecutionQueued(job *plugin_pb.JobSpec) {
})
}
// bindQueuedJobWorker records the home worker while the job stays pending/queued.
func (r *Plugin) bindQueuedJobWorker(job *plugin_pb.JobSpec, workerID string) {
if job == nil || strings.TrimSpace(job.JobId) == "" || strings.TrimSpace(workerID) == "" {
return
}
now := time.Now().UTC()
r.jobsMu.Lock()
tracked := r.jobs[job.JobId]
if tracked == nil {
r.jobsMu.Unlock()
return
}
if isTerminalTrackedJobState(tracked.State) {
r.jobsMu.Unlock()
return
}
tracked.WorkerID = workerID
tracked.Message = "queued on worker " + workerID
tracked.UpdatedAt = timeToPtr(now)
snap := cloneTrackedJob(*tracked)
r.dirtyJobs = true
r.jobsMu.Unlock()
r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) {
detail.WorkerID = snap.WorkerID
detail.Message = snap.Message
detail.UpdatedAt = snap.UpdatedAt
})
}
func (r *Plugin) cancelQueuedJob(job *plugin_pb.JobSpec, cause error) {
reason := "job canceled"
if cause != nil {
+206 -125
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"sort"
"strings"
"sync"
"sync/atomic"
@@ -28,12 +29,18 @@ const (
defaultScheduledMaxResults int32 = 1000
defaultScheduledExecutionConcurrency = 1
defaultScheduledPerWorkerConcurrency = 1
maxScheduledExecutionConcurrency = 128
defaultScheduledRetryBackoff = 5 * time.Second
defaultClusterContextTimeout = 10 * time.Second
defaultWaitingBacklogFloor = 8
defaultWaitingBacklogMultiplier = 4
maxEstimatedRuntimeCap = 8 * time.Hour
// Hold a job on replica-host workers until their reservation load
// reaches this; 0 disables the pin and spills immediately.
defaultLocalQueuePin = 5
// Wait this long after a home worker disappears before stealing
// its queue; 0 steals immediately. Covers a reconnect blink.
defaultOrphanRehomeGrace = 30 * time.Second
maxScheduledExecutionConcurrency = 128
defaultScheduledRetryBackoff = 5 * time.Second
defaultClusterContextTimeout = 10 * time.Second
defaultWaitingBacklogFloor = 8
defaultWaitingBacklogMultiplier = 4
maxEstimatedRuntimeCap = 8 * time.Hour
// How long started jobs may drain past the run window close.
scheduledExecutionDrainGrace = 30 * time.Minute
)
@@ -49,6 +56,12 @@ type schedulerPolicy struct {
PerWorkerConcurrency int
RetryLimit int
ExecutorReserveBackoff time.Duration
// LocalQueuePin is K: do not spill off replica-host workers while
// min(load(pref)) < K. Zero is the old immediate-spill path.
LocalQueuePin int
// OrphanRehomeGrace is how long a missing home's queue is held
// before steal-without-K. Zero steals as soon as the worker is gone.
OrphanRehomeGrace time.Duration
}
// laneSchedulerLoop is the main scheduling goroutine for a single lane.
@@ -466,6 +479,8 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
PerWorkerConcurrency: int(adminRuntime.PerWorkerExecutionConcurrency),
RetryLimit: int(adminRuntime.RetryLimit),
ExecutorReserveBackoff: 200 * time.Millisecond,
LocalQueuePin: defaultLocalQueuePin,
OrphanRehomeGrace: defaultOrphanRehomeGrace,
}
if policy.DetectionInterval < r.schedulerTick {
@@ -496,6 +511,9 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
if policy.ExecutionTimeout < defaultScheduledExecutionTimeout {
policy.ExecutionTimeout = defaultScheduledExecutionTimeout
}
if policy.LocalQueuePin < 0 {
policy.LocalQueuePin = 0
}
return policy, true, nil
}
@@ -813,19 +831,22 @@ func (r *Plugin) dispatchScheduledProposals(
// whole dispatch lets manual shell operations interleave between jobs.
jobsRequireLock := LaneRequiresLock(JobTypeLane(jobType))
jobQueue := make(chan *plugin_pb.JobSpec, len(proposals))
jobs := make([]*plugin_pb.JobSpec, 0, len(proposals))
for index, proposal := range proposals {
job := buildScheduledJobSpec(jobType, proposal, index)
r.trackExecutionQueued(job)
select {
case <-r.shutdownCh:
close(jobQueue)
return 0, 0, 0
for _, job := range jobs {
r.cancelQueuedJob(job, errSchedulerShutdown)
}
return 0, 0, len(jobs)
default:
jobQueue <- job
}
job := buildScheduledJobSpec(jobType, proposal, index)
r.trackExecutionQueued(job)
jobs = append(jobs, job)
}
close(jobQueue)
queues := r.assignJobsToWorkerQueues(jobType, jobs, policy)
var wg sync.WaitGroup
var statsMu sync.Mutex
@@ -842,130 +863,110 @@ func (r *Plugin) dispatchScheduledProposals(
wg.Add(1)
go func() {
defer wg.Done()
jobLoop:
for job := range jobQueue {
for {
select {
case <-r.shutdownCh:
return
default:
}
if ctx.Err() != nil {
r.cancelQueuedJob(job, ctx.Err())
statsMu.Lock()
canceledCount++
statsMu.Unlock()
return
}
if queues.isEmpty() {
return
}
item, targetID := r.takeQueuedJob(queues, jobType, policy)
if item == nil {
if queues.isEmpty() {
return
}
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
return
}
continue
}
if ctx.Err() != nil {
queues.pushFront(item)
return
}
executor, release, reserveErr := r.reserveExecutorByID(jobType, policy, targetID)
if reserveErr != nil {
queues.pushFront(item)
if ctx.Err() != nil {
return
}
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
return
}
continue
}
for {
select {
case <-r.shutdownCh:
return
default:
var releaseJobLock func()
if jobsRequireLock {
var lockErr error
releaseJobLock, lockErr = r.acquireAdminLock(fmt.Sprintf("plugin scheduler %s job %s", jobType, item.job.JobId))
if lockErr != nil {
release()
statsMu.Lock()
errorCount++
statsMu.Unlock()
r.appendActivity(JobActivity{
JobID: item.job.JobId,
JobType: item.job.JobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled execution lock failed: %v", lockErr),
Stage: "failed",
OccurredAt: timeToPtr(time.Now().UTC()),
})
continue
}
if ctx.Err() != nil {
r.cancelQueuedJob(job, ctx.Err())
releaseJobLock()
release()
queues.pushFront(item)
return
}
}
err := r.executeScheduledJobWithExecutor(ctx, executor, item.job, clusterContext, policy)
if releaseJobLock != nil {
releaseJobLock()
}
release()
if errors.Is(err, errExecutorAtCapacity) {
queues.pushFront(item)
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
return
}
continue
}
if err != nil {
if ctx.Err() != nil || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
r.cancelQueuedJob(item.job, err)
statsMu.Lock()
canceledCount++
statsMu.Unlock()
continue jobLoop
}
executor, release, reserveErr := r.reserveScheduledExecutor(ctx, jobType, policy)
if reserveErr != nil {
if ctx.Err() != nil {
r.cancelQueuedJob(job, ctx.Err())
statsMu.Lock()
canceledCount++
statsMu.Unlock()
continue jobLoop
}
statsMu.Lock()
errorCount++
statsMu.Unlock()
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled execution reservation failed: %v", reserveErr),
Stage: "failed",
OccurredAt: timeToPtr(time.Now().UTC()),
})
break
}
var releaseJobLock func()
if jobsRequireLock {
var lockErr error
releaseJobLock, lockErr = r.acquireAdminLock(fmt.Sprintf("plugin scheduler %s job %s", jobType, job.JobId))
if lockErr != nil {
release()
statsMu.Lock()
errorCount++
statsMu.Unlock()
r.appendActivity(JobActivity{
JobID: job.JobId,
JobType: job.JobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled execution lock failed: %v", lockErr),
Stage: "failed",
OccurredAt: timeToPtr(time.Now().UTC()),
})
break
}
// The lock may have been held by an operator for a
// while; re-check the execution window.
if ctx.Err() != nil {
releaseJobLock()
release()
r.cancelQueuedJob(job, ctx.Err())
statsMu.Lock()
canceledCount++
statsMu.Unlock()
continue jobLoop
}
}
err := r.executeScheduledJobWithExecutor(ctx, executor, job, clusterContext, policy)
if releaseJobLock != nil {
releaseJobLock()
}
release()
if errors.Is(err, errExecutorAtCapacity) {
r.trackExecutionQueued(job)
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
return
}
continue
}
if err != nil {
if ctx.Err() != nil || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
r.cancelQueuedJob(job, err)
statsMu.Lock()
canceledCount++
statsMu.Unlock()
continue jobLoop
}
statsMu.Lock()
errorCount++
statsMu.Unlock()
r.appendActivity(JobActivity{
JobID: job.JobId,
JobType: job.JobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled execution failed: %v", err),
Stage: "failed",
OccurredAt: timeToPtr(time.Now().UTC()),
})
break
}
statsMu.Lock()
successCount++
errorCount++
statsMu.Unlock()
break
r.appendActivity(JobActivity{
JobID: item.job.JobId,
JobType: item.job.JobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled execution failed: %v", err),
Stage: "failed",
OccurredAt: timeToPtr(time.Now().UTC()),
})
continue
}
statsMu.Lock()
successCount++
statsMu.Unlock()
}
}()
}
@@ -976,8 +977,8 @@ func (r *Plugin) dispatchScheduledProposals(
if drainErr == nil {
drainErr = errSchedulerShutdown
}
for job := range jobQueue {
r.cancelQueuedJob(job, drainErr)
for _, item := range queues.drain() {
r.cancelQueuedJob(item.job, drainErr)
canceledCount++
}
@@ -988,6 +989,7 @@ func (r *Plugin) reserveScheduledExecutor(
ctx context.Context,
jobType string,
policy schedulerPolicy,
preferredHosts []string,
) (*WorkerSession, func(), error) {
if ctx == nil {
ctx = context.Background()
@@ -1015,7 +1017,7 @@ func (r *Plugin) reserveScheduledExecutor(
return nil, nil, fmt.Errorf("timed out waiting for executor capacity for %s", jobType)
}
executors, err := r.registry.ListExecutors(jobType)
executors, err := r.registry.ListExecutors(jobType, preferredHosts...)
if err != nil {
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
if ctx.Err() != nil {
@@ -1026,11 +1028,19 @@ func (r *Plugin) reserveScheduledExecutor(
continue
}
for _, executor := range executors {
release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy)
if !ok {
continue
pref, rest := partitionByPreferredHost(executors, preferredHosts)
lane := JobTypeLane(jobType)
loadOf := func(e *WorkerSession) int { return r.executorReservationLoad(e, lane) }
if len(pref) > 0 {
sortExecutorsByLoad(pref, loadOf)
if executor, release, ok := r.tryReserveFirst(pref, jobType, policy); ok {
return executor, release, nil
}
}
sortExecutorsByLoad(rest, loadOf)
if executor, release, ok := r.tryReserveFirst(rest, jobType, policy); ok {
return executor, release, nil
}
@@ -1051,6 +1061,75 @@ func (r *Plugin) tryReserveExecutorCapacity(
return r.tryReserveExecutorCapacityForLane(executor, jobType, policy, JobTypeLane(jobType))
}
func (r *Plugin) tryReserveFirst(executors []*WorkerSession, jobType string, policy schedulerPolicy) (*WorkerSession, func(), bool) {
for _, executor := range executors {
release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy)
if ok {
return executor, release, true
}
}
return nil, nil, false
}
func (r *Plugin) reserveExecutorByID(jobType string, policy schedulerPolicy, workerID string) (*WorkerSession, func(), error) {
workerID = strings.TrimSpace(workerID)
if workerID == "" {
return nil, nil, fmt.Errorf("executor id is empty")
}
executors, err := r.registry.ListExecutors(jobType)
if err != nil {
return nil, nil, err
}
for _, executor := range executors {
if executor == nil || executor.WorkerID != workerID {
continue
}
release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy)
if !ok {
return nil, nil, errExecutorAtCapacity
}
return executor, release, nil
}
return nil, nil, fmt.Errorf("executor %s is not available for %s", workerID, jobType)
}
func (r *Plugin) executorReservationLoad(executor *WorkerSession, lane SchedulerLane) int {
if executor == nil {
return 0
}
load := 0
if executor.Heartbeat != nil && executor.Heartbeat.ExecutionSlotsUsed > 0 {
load = int(executor.Heartbeat.ExecutionSlotsUsed)
}
workerID := strings.TrimSpace(executor.WorkerID)
if ls := r.lanes[lane]; ls != nil {
ls.execMu.Lock()
load += ls.execRes[workerID]
ls.execMu.Unlock()
return load
}
r.schedulerExecMu.Lock()
load += r.schedulerExecReservations[workerID]
r.schedulerExecMu.Unlock()
return load
}
func sortExecutorsByLoad(executors []*WorkerSession, load func(*WorkerSession) int) {
sort.SliceStable(executors, func(i, j int) bool {
if executors[i] == nil {
return false
}
if executors[j] == nil {
return true
}
li, lj := load(executors[i]), load(executors[j])
if li != lj {
return li < lj
}
return executors[i].WorkerID < executors[j].WorkerID
})
}
// tryReserveExecutorCapacityForLane reserves an execution slot on the
// per-lane reservation pool so that lanes cannot starve each other.
func (r *Plugin) tryReserveExecutorCapacityForLane(
@@ -1577,6 +1656,8 @@ func (r *Plugin) DispatchProposals(
ExecutionTimeout: defaultScheduledExecutionTimeout,
RetryBackoff: defaultScheduledRetryBackoff,
ExecutorReserveBackoff: 200 * time.Millisecond,
LocalQueuePin: defaultLocalQueuePin,
OrphanRehomeGrace: defaultOrphanRehomeGrace,
}
}
+58 -5
View File
@@ -61,6 +61,12 @@ func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) {
if policy.JobTypeMaxRuntime != 30*time.Minute {
t.Fatalf("unexpected max runtime: got=%v", policy.JobTypeMaxRuntime)
}
if policy.LocalQueuePin != defaultLocalQueuePin {
t.Fatalf("unexpected local queue pin: got=%d", policy.LocalQueuePin)
}
if policy.OrphanRehomeGrace != defaultOrphanRehomeGrace {
t.Fatalf("unexpected orphan rehome grace: got=%v", policy.OrphanRehomeGrace)
}
}
func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T) {
@@ -138,13 +144,13 @@ func TestReserveScheduledExecutorRespectsPerWorkerLimit(t *testing.T) {
ExecutorReserveBackoff: time.Millisecond,
}
executor1, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
executor1, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
if err != nil {
t.Fatalf("reserve executor 1: %v", err)
}
defer release1()
executor2, release2, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
executor2, release2, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
if err != nil {
t.Fatalf("reserve executor 2: %v", err)
}
@@ -266,7 +272,7 @@ func TestReserveScheduledExecutorTimesOutWhenNoExecutor(t *testing.T) {
start := time.Now()
pluginSvc.Shutdown()
_, _, err = pluginSvc.reserveScheduledExecutor(context.Background(), "missing-job-type", policy)
_, _, err = pluginSvc.reserveScheduledExecutor(context.Background(), "missing-job-type", policy, nil)
if err == nil {
t.Fatalf("expected reservation shutdown error")
}
@@ -297,7 +303,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
ExecutorReserveBackoff: 5 * time.Millisecond,
}
_, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
_, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
if err != nil {
t.Fatalf("reserve executor 1: %v", err)
}
@@ -308,7 +314,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
}
secondReserveCh := make(chan reserveResult, 1)
go func() {
_, release2, reserveErr := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
_, release2, reserveErr := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
if release2 != nil {
release2()
}
@@ -992,3 +998,50 @@ func TestLaneStatus_LifecycleNextDetectionSurvivesDefaultLanePrune(t *testing.T)
status.NextDetectionAt, expected)
}
}
func TestReserveWaterfillsPreferredHosts(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
ids := []string{"worker-a", "worker-b", "worker-c", "worker-d"}
for _, id := range ids {
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: id,
Address: "10.0.0.1:9334",
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 8},
},
})
}
policy := schedulerPolicy{
PerWorkerConcurrency: 8,
ExecutorReserveBackoff: time.Millisecond,
LocalQueuePin: 5,
}
preferred := []string{"10.0.0.1:8080"}
seen := map[string]int{}
var releases []func()
defer func() {
for _, release := range releases {
release()
}
}()
for i := 0; i < 4; i++ {
exec, release, err := pluginSvc.reserveScheduledExecutor(context.Background(), "erasure_coding", policy, preferred)
if err != nil {
t.Fatalf("reserve %d: %v", i, err)
}
releases = append(releases, release)
seen[exec.WorkerID]++
}
if len(seen) != 4 {
t.Fatalf("want one job per worker, got %v", seen)
}
}
+23 -5
View File
@@ -277,14 +277,24 @@ func (r *Registry) PickDetector(jobType string) (*WorkerSession, error) {
}
// PickExecutor picks one executor worker for a job type.
func (r *Registry) PickExecutor(jobType string) (*WorkerSession, error) {
return r.pickByKind(jobType, false)
// preferredHosts, when set, ranks workers whose Hello Address host matches a
// replica/source host ahead of others (same-machine EC pull). Empty means the
// previous slots+round-robin order.
func (r *Registry) PickExecutor(jobType string, preferredHosts ...string) (*WorkerSession, error) {
list, err := r.ListExecutors(jobType, preferredHosts...)
if err != nil {
return nil, err
}
return list[0], nil
}
// ListExecutors returns sorted executor candidates for one job type.
// Ordering is by most available execution slots, then lexical worker ID.
// The top tie group is rotated round-robin to prevent sticky assignment.
func (r *Registry) ListExecutors(jobType string) ([]*WorkerSession, error) {
// preferredHosts, when any worker matches, are listed first (then the rest);
// rotation applies only inside the preferred group so a local worker is not
// rotated away in favor of a remote one.
func (r *Registry) ListExecutors(jobType string, preferredHosts ...string) ([]*WorkerSession, error) {
r.mu.Lock()
defer r.mu.Unlock()
@@ -293,8 +303,16 @@ func (r *Registry) ListExecutors(jobType string) ([]*WorkerSession, error) {
return nil, fmt.Errorf("no executor worker available for job_type=%s", jobType)
}
sortByKind(candidates, jobType, false)
r.rotateTopCandidatesLocked(candidates, jobType, false)
pref, rest := partitionByPreferredHost(candidates, preferredHosts)
if len(pref) > 0 {
sortByKind(pref, jobType, false)
r.rotateTopCandidatesLocked(pref, jobType, false)
sortByKind(rest, jobType, false)
candidates = append(pref, rest...)
} else {
sortByKind(candidates, jobType, false)
r.rotateTopCandidatesLocked(candidates, jobType, false)
}
out := make([]*WorkerSession, 0, len(candidates))
for _, candidate := range candidates {
+67
View File
@@ -401,3 +401,70 @@ func TestRegistryHasCapableWorkerIgnoresNilCapability(t *testing.T) {
t.Fatalf("expected HasCapableWorker to return false when capability has no detect or execute")
}
}
func TestListExecutorsPrefersMatchingReplicaHost(t *testing.T) {
t.Parallel()
r := NewRegistry()
r.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: "worker-remote",
Address: "10.87.0.4:33646",
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 8},
},
})
r.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: "worker-local",
Address: "10.87.0.3:33646",
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 1},
},
})
r.UpdateHeartbeat("worker-remote", &plugin_pb.WorkerHeartbeat{
WorkerId: "worker-remote", ExecutionSlotsUsed: 0, ExecutionSlotsTotal: 8,
})
r.UpdateHeartbeat("worker-local", &plugin_pb.WorkerHeartbeat{
WorkerId: "worker-local", ExecutionSlotsUsed: 0, ExecutionSlotsTotal: 1,
})
// Without preference the emptier remote worker wins on slots.
plain, err := r.ListExecutors("erasure_coding")
if err != nil {
t.Fatalf("ListExecutors: %v", err)
}
if plain[0].WorkerID != "worker-remote" {
t.Fatalf("without preference want remote first, got %s", plain[0].WorkerID)
}
// Replica on :8080 of the same host as worker-local.
pref, err := r.ListExecutors("erasure_coding", "10.87.0.3:8080")
if err != nil {
t.Fatalf("ListExecutors preferred: %v", err)
}
if pref[0].WorkerID != "worker-local" {
t.Fatalf("preferred host want worker-local first, got %s", pref[0].WorkerID)
}
if pref[1].WorkerID != "worker-remote" {
t.Fatalf("remote should still be fallback, got %s", pref[1].WorkerID)
}
}
func TestJobReplicaHostsFromSourcesAndReplicaServers(t *testing.T) {
t.Parallel()
job := &plugin_pb.JobSpec{
Parameters: map[string]*plugin_pb.ConfigValue{
"source_server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "10.87.0.1:8080"},
},
"replica_servers": {
Kind: &plugin_pb.ConfigValue_StringList{
StringList: &plugin_pb.StringList{Values: []string{"10.87.0.1:8080", "10.87.0.2:8081"}},
},
},
},
}
got := JobReplicaHosts(job)
if len(got) != 2 {
t.Fatalf("got %v", got)
}
}
+317
View File
@@ -0,0 +1,317 @@
package plugin
import (
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// queuedJob is one scheduled spec parked on a home worker's admin-side queue.
type queuedJob struct {
job *plugin_pb.JobSpec
home string
}
// workerJobQueues holds per-worker FIFOs for one dispatch batch.
// Home assignment is done once up front; run goroutines only pop/steal.
// leftover is the homeless FIFO: replica hostnames with no currently
// registered worker. Orphan grace does not apply here.
type workerJobQueues struct {
mu sync.Mutex
q map[string][]*queuedJob
leftover []*queuedJob
remaining int
missingSince map[string]time.Time
homes map[string]struct{}
}
func newWorkerJobQueues() *workerJobQueues {
return &workerJobQueues{
q: make(map[string][]*queuedJob),
missingSince: make(map[string]time.Time),
homes: make(map[string]struct{}),
}
}
func (qs *workerJobQueues) hasHome(workerID string) bool {
if qs == nil || workerID == "" {
return false
}
_, ok := qs.homes[workerID]
return ok
}
func (qs *workerJobQueues) push(home string, job *plugin_pb.JobSpec) {
if qs == nil || job == nil {
return
}
item := &queuedJob{job: job, home: home}
qs.mu.Lock()
defer qs.mu.Unlock()
qs.pushLocked(item, false)
}
func (qs *workerJobQueues) pushFront(item *queuedJob) {
if qs == nil || item == nil {
return
}
qs.mu.Lock()
defer qs.mu.Unlock()
qs.pushLocked(item, true)
}
func (qs *workerJobQueues) pushLocked(item *queuedJob, front bool) {
qs.remaining++
if item.home == "" {
if front {
qs.leftover = append([]*queuedJob{item}, qs.leftover...)
return
}
qs.leftover = append(qs.leftover, item)
return
}
if front {
qs.q[item.home] = append([]*queuedJob{item}, qs.q[item.home]...)
return
}
qs.q[item.home] = append(qs.q[item.home], item)
}
func (qs *workerJobQueues) isEmpty() bool {
if qs == nil {
return true
}
qs.mu.Lock()
defer qs.mu.Unlock()
return qs.remaining == 0
}
func (qs *workerJobQueues) queueLen(workerID string) int {
if qs == nil {
return 0
}
qs.mu.Lock()
defer qs.mu.Unlock()
return len(qs.q[workerID])
}
func (qs *workerJobQueues) leftoverLen() int {
if qs == nil {
return 0
}
qs.mu.Lock()
defer qs.mu.Unlock()
return len(qs.leftover)
}
func (qs *workerJobQueues) drain() []*queuedJob {
if qs == nil {
return nil
}
qs.mu.Lock()
defer qs.mu.Unlock()
out := make([]*queuedJob, 0, qs.remaining)
for id, qq := range qs.q {
out = append(out, qq...)
qs.q[id] = nil
}
out = append(out, qs.leftover...)
qs.leftover = nil
qs.remaining = 0
return out
}
func popFront(qq []*queuedJob) (*queuedJob, []*queuedJob) {
item := qq[0]
if len(qq) == 1 {
return item, nil
}
return item, qq[1:]
}
// assignJobsToWorkerQueues parks each job on the least-loaded replica-host
// worker. A home exists only if a currently registered worker's -address
// hostname matches a replica host. Otherwise the job is homeless: global
// leftover FIFO, no fake home, no K, no orphan grace. Single-threaded.
func (r *Plugin) assignJobsToWorkerQueues(
jobType string,
jobs []*plugin_pb.JobSpec,
_ schedulerPolicy,
) *workerJobQueues {
qs := newWorkerJobQueues()
executors, err := r.registry.ListExecutors(jobType)
lane := JobTypeLane(jobType)
queued := make(map[string]int, len(executors))
for _, job := range jobs {
if job == nil {
continue
}
if err != nil || len(executors) == 0 {
qs.push("", job)
continue
}
pref, _ := partitionByPreferredHost(executors, JobReplicaHosts(job))
if len(pref) == 0 {
// Never registered for that hostname: homeless, not an orphan.
qs.push("", job)
continue
}
for _, e := range pref {
if e != nil && e.WorkerID != "" {
qs.homes[e.WorkerID] = struct{}{}
}
}
sortExecutorsByLoad(pref, func(e *WorkerSession) int {
return r.executorReservationLoad(e, lane) + queued[e.WorkerID]
})
home := pref[0].WorkerID
qs.push(home, job)
queued[home]++
r.bindQueuedJobWorker(job, home)
}
return qs
}
// takeQueuedJob pops a job for a free worker.
// Home worker (hostname matched a replica / has a home-queue identity):
// 1. own home queue
// 2. steal from another home if len>=K, or orphan past OrphanRehomeGrace
// 3. homeless leftover FIFO
// No-home worker (no matching replica, no home-queue identity):
// 1. homeless leftover FIFO
// 2. only if leftover empty: steal (K / orphan), same rules
func (r *Plugin) takeQueuedJob(
qs *workerJobQueues,
jobType string,
policy schedulerPolicy,
) (item *queuedJob, targetID string) {
if qs == nil {
return nil, ""
}
executors, err := r.registry.ListExecutors(jobType)
if err != nil || len(executors) == 0 {
return nil, ""
}
lane := JobTypeLane(jobType)
pin := policy.LocalQueuePin
defer func() {
if item != nil && targetID != "" {
r.bindQueuedJobWorker(item.job, targetID)
}
}()
qs.mu.Lock()
defer qs.mu.Unlock()
if qs.remaining == 0 {
return nil, ""
}
var bestHome *WorkerSession
bestLoad := 0
for _, exec := range executors {
if len(qs.q[exec.WorkerID]) == 0 {
continue
}
limit := schedulerWorkerExecutionLimit(exec, jobType, policy)
if limit <= 0 {
continue
}
load := r.executorReservationLoad(exec, lane)
if load >= limit {
continue
}
if bestHome == nil || load < bestLoad || (load == bestLoad && exec.WorkerID < bestHome.WorkerID) {
bestHome = exec
bestLoad = load
}
}
if bestHome != nil {
id := bestHome.WorkerID
item, qs.q[id] = popFront(qs.q[id])
qs.remaining--
return item, id
}
var thief *WorkerSession
for _, exec := range executors {
limit := schedulerWorkerExecutionLimit(exec, jobType, policy)
if limit <= 0 {
continue
}
if r.executorReservationLoad(exec, lane) >= limit {
continue
}
if thief == nil || exec.WorkerID < thief.WorkerID {
thief = exec
}
}
if thief == nil {
return nil, ""
}
takeLeftover := func() (*queuedJob, string) {
if len(qs.leftover) == 0 {
return nil, ""
}
item, qs.leftover = popFront(qs.leftover)
qs.remaining--
return item, thief.WorkerID
}
if !qs.hasHome(thief.WorkerID) {
if item, id := takeLeftover(); item != nil {
return item, id
}
}
live := make(map[string]struct{}, len(executors))
for _, exec := range executors {
live[exec.WorkerID] = struct{}{}
delete(qs.missingSince, exec.WorkerID)
}
now := time.Now()
grace := policy.OrphanRehomeGrace
victimID := ""
victimLen := 0
victimOrphan := false
for id, qq := range qs.q {
if len(qq) == 0 {
delete(qs.missingSince, id)
continue
}
_, isLive := live[id]
orphan := !isLive
if isLive {
if pin > 0 && len(qq) < pin {
continue
}
} else {
if _, ok := qs.missingSince[id]; !ok {
qs.missingSince[id] = now
}
if grace > 0 && now.Sub(qs.missingSince[id]) < grace {
continue
}
}
better := victimID == ""
if !better && orphan != victimOrphan {
better = orphan
}
if !better && orphan == victimOrphan {
better = len(qq) > victimLen || (len(qq) == victimLen && id < victimID)
}
if better {
victimID = id
victimLen = len(qq)
victimOrphan = orphan
}
}
if victimID != "" {
item, qs.q[victimID] = popFront(qs.q[victimID])
qs.remaining--
return item, thief.WorkerID
}
return takeLeftover()
}
+600
View File
@@ -0,0 +1,600 @@
package plugin
import (
"fmt"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
func replicaJob(id, source string) *plugin_pb.JobSpec {
return &plugin_pb.JobSpec{
JobId: id,
JobType: "erasure_coding",
Summary: id,
DedupeKey: id,
Parameters: map[string]*plugin_pb.ConfigValue{
"source_server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: source},
},
},
}
}
func registerECWorkers(t *testing.T, pluginSvc *Plugin, workers map[string]string) {
t.Helper()
for id, addr := range workers {
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: id,
Address: addr,
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 8},
},
})
}
}
func TestAssignJobsWaterfillsPreferredWorkers(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
registerECWorkers(t, pluginSvc, map[string]string{
"worker-a": "10.0.0.1:9334",
"worker-b": "10.0.0.2:9334",
"worker-c": "10.0.0.3:9334",
"worker-d": "10.0.0.4:9334",
})
jobs := []*plugin_pb.JobSpec{
replicaJob("j1", "10.0.0.1:8080"),
replicaJob("j2", "10.0.0.1:8080"),
replicaJob("j3", "10.0.0.1:8080"),
replicaJob("j4", "10.0.0.1:8080"),
}
// All four replica IPs listed so every worker is preferred.
for _, job := range jobs {
job.Parameters["replica_servers"] = &plugin_pb.ConfigValue{
Kind: &plugin_pb.ConfigValue_StringList{StringList: &plugin_pb.StringList{
Values: []string{"10.0.0.1:8080", "10.0.0.2:8080", "10.0.0.3:8080", "10.0.0.4:8080"},
}},
}
}
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, schedulerPolicy{LocalQueuePin: 5})
if qs.queueLen("worker-a")+qs.queueLen("worker-b")+qs.queueLen("worker-c")+qs.queueLen("worker-d") != 4 {
t.Fatalf("expected 4 assigned jobs")
}
for _, id := range []string{"worker-a", "worker-b", "worker-c", "worker-d"} {
if qs.queueLen(id) != 1 {
t.Fatalf("want 1 job on %s, got %d", id, qs.queueLen(id))
}
}
}
func TestAssignAndTakeDoesNotStarveOtherHomes(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
registerECWorkers(t, pluginSvc, map[string]string{
"worker-local": "10.0.0.1:9334",
"worker-remote": "10.0.0.2:9334",
})
jobs := []*plugin_pb.JobSpec{
replicaJob("local-1", "10.0.0.1:8080"),
replicaJob("local-2", "10.0.0.1:8080"),
replicaJob("remote-1", "10.0.0.2:8080"),
}
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
if qs.queueLen("worker-local") != 2 || qs.queueLen("worker-remote") != 1 {
t.Fatalf("homes local=%d remote=%d", qs.queueLen("worker-local"), qs.queueLen("worker-remote"))
}
_, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
_, release1, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, target1)
if err != nil {
t.Fatalf("reserve 1: %v", err)
}
defer release1()
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if item2 == nil || target2 != "worker-remote" {
t.Fatalf("second take should start remote home, got job=%v target=%s", item2, target2)
}
}
func TestTakeStealsOnlyWhenHomeQueueReachesK(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
registerECWorkers(t, pluginSvc, map[string]string{
"worker-local": "10.0.0.1:9334",
"worker-remote": "10.0.0.2:9334",
})
jobs := make([]*plugin_pb.JobSpec, 0, 6)
for i := 0; i < 6; i++ {
jobs = append(jobs, replicaJob(fmt.Sprintf("j%d", i), "10.0.0.1:8080"))
}
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
if qs.queueLen("worker-local") != 6 {
t.Fatalf("all jobs should home on local, got %d", qs.queueLen("worker-local"))
}
_, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if target != "worker-local" {
t.Fatalf("first take want local, got %s", target)
}
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
if err != nil {
t.Fatalf("reserve local: %v", err)
}
defer release()
// 5 still queued on local (>=K) and local has no slot → steal to remote.
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if stolen == nil || stealTarget != "worker-remote" {
t.Fatalf("expected steal to remote, got job=%v target=%s", stolen, stealTarget)
}
if qs.queueLen("worker-local") != 4 {
t.Fatalf("after steal want 4 left on local, got %d", qs.queueLen("worker-local"))
}
// 4 < K: do not steal again.
if item, tgt := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy); item != nil {
t.Fatalf("pin should hold remaining jobs, got target=%s", tgt)
}
}
func TestTakeStealsImmediatelyWhenPinDisabled(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
registerECWorkers(t, pluginSvc, map[string]string{
"worker-local": "10.0.0.1:9334",
"worker-remote": "10.0.0.2:9334",
})
jobs := []*plugin_pb.JobSpec{
replicaJob("a", "10.0.0.1:8080"),
replicaJob("b", "10.0.0.1:8080"),
}
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 0}
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
_, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if target != "worker-local" {
t.Fatalf("first take want local, got %s", target)
}
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
if err != nil {
t.Fatalf("reserve local: %v", err)
}
defer release()
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if stolen == nil || stealTarget != "worker-remote" {
t.Fatalf("K=0 should steal immediately, got job=%v target=%s", stolen, stealTarget)
}
}
func TestTakeStealsOrphanedHomeWithoutK(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
registerECWorkers(t, pluginSvc, map[string]string{
"worker-local": "10.0.0.1:9334",
"worker-remote": "10.0.0.2:9334",
})
jobs := []*plugin_pb.JobSpec{
replicaJob("o1", "10.0.0.1:8080"),
replicaJob("o2", "10.0.0.1:8080"),
}
policy := schedulerPolicy{
PerWorkerConcurrency: 1,
LocalQueuePin: 5,
OrphanRehomeGrace: 30 * time.Second,
}
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
if qs.queueLen("worker-local") != 2 {
t.Fatalf("want 2 on local, got %d", qs.queueLen("worker-local"))
}
pluginSvc.registry.Remove("worker-local")
if item, tgt := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy); item != nil {
t.Fatalf("within grace should hold orphan queue, got target=%s", tgt)
}
if qs.queueLen("worker-local") != 2 {
t.Fatalf("held queue changed: %d", qs.queueLen("worker-local"))
}
qs.mu.Lock()
qs.missingSince["worker-local"] = time.Now().Add(-time.Minute)
qs.mu.Unlock()
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if stolen == nil || stealTarget != "worker-remote" {
t.Fatalf("after grace should steal without K, got job=%v target=%s", stolen, stealTarget)
}
if qs.queueLen("worker-local") != 1 {
t.Fatalf("want 1 orphan left, got %d", qs.queueLen("worker-local"))
}
stolen2, stealTarget2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if stolen2 == nil || stealTarget2 != "worker-remote" {
t.Fatalf("second orphan should also move, got job=%v target=%s", stolen2, stealTarget2)
}
}
func TestTakeKeepsOrphanWhenHomeReconnectsDuringGrace(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
registerECWorkers(t, pluginSvc, map[string]string{
"worker-local": "10.0.0.1:9334",
"worker-remote": "10.0.0.2:9334",
})
jobs := []*plugin_pb.JobSpec{replicaJob("blink", "10.0.0.1:8080")}
policy := schedulerPolicy{
PerWorkerConcurrency: 1,
LocalQueuePin: 5,
OrphanRehomeGrace: 30 * time.Second,
}
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
pluginSvc.registry.Remove("worker-local")
if item, _ := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy); item != nil {
t.Fatal("expected hold during grace")
}
registerECWorkers(t, pluginSvc, map[string]string{"worker-local": "10.0.0.1:9334"})
item, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if item == nil || target != "worker-local" {
t.Fatalf("reconnected home should run its queue, got job=%v target=%s", item, target)
}
}
func TestTakeStealsOrphanImmediatelyWhenGraceDisabled(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
registerECWorkers(t, pluginSvc, map[string]string{
"worker-local": "10.0.0.1:9334",
"worker-remote": "10.0.0.2:9334",
})
jobs := []*plugin_pb.JobSpec{replicaJob("o1", "10.0.0.1:8080")}
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5, OrphanRehomeGrace: 0}
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
pluginSvc.registry.Remove("worker-local")
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if stolen == nil || stealTarget != "worker-remote" {
t.Fatalf("grace=0 should steal immediately, got job=%v target=%s", stolen, stealTarget)
}
}
func TestBindQueuedJobWorkerKeepsPendingState(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
job := replicaJob("bind-1", "10.0.0.1:8080")
pluginSvc.trackExecutionQueued(job)
pluginSvc.bindQueuedJobWorker(job, "worker-local")
pluginSvc.jobsMu.Lock()
tracked := pluginSvc.jobs[job.JobId]
pluginSvc.jobsMu.Unlock()
if tracked == nil {
t.Fatal("missing tracked job")
}
if tracked.State != "job_state_pending" {
t.Fatalf("state=%s", tracked.State)
}
if tracked.Stage != "queued" {
t.Fatalf("stage=%s", tracked.Stage)
}
if tracked.WorkerID != "worker-local" {
t.Fatalf("worker=%s", tracked.WorkerID)
}
}
func TestHomelessAllRemoteWorkersUseFIFOWithoutPinOrGrace(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
// Workers never registered on the replica hostname — homeless, not orphan.
registerECWorkers(t, pluginSvc, map[string]string{
"worker-a": "10.0.0.5:9334",
"worker-b": "10.0.0.6:9334",
})
jobs := []*plugin_pb.JobSpec{
replicaJob("h1", "10.0.0.1:8080"),
replicaJob("h2", "10.0.0.1:8080"),
replicaJob("h3", "10.0.0.1:8080"),
}
policy := schedulerPolicy{
PerWorkerConcurrency: 1,
LocalQueuePin: 5,
OrphanRehomeGrace: 30 * time.Second,
}
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
if qs.queueLen("worker-a") != 0 || qs.queueLen("worker-b") != 0 {
t.Fatalf("homeless jobs must not get a fake home, a=%d b=%d", qs.queueLen("worker-a"), qs.queueLen("worker-b"))
}
if qs.leftoverLen() != 3 {
t.Fatalf("want 3 homeless FIFO, got %d", qs.leftoverLen())
}
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if item1 == nil || item1.job.JobId != "h1" {
t.Fatalf("FIFO first want h1, got %v", item1)
}
_, release1, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, target1)
if err != nil {
t.Fatalf("reserve 1: %v", err)
}
defer release1()
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if item2 == nil || item2.job.JobId != "h2" {
t.Fatalf("FIFO second want h2 immediately, got %v", item2)
}
if target2 == "" || target2 == target1 {
t.Fatalf("second take should hit the other idle worker, t1=%s t2=%s", target1, target2)
}
if qs.leftoverLen() != 1 {
t.Fatalf("want 1 homeless left, got %d", qs.leftoverLen())
}
}
func TestMixedReplicaLocalJobsStayHomeBeforeHomeless(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
registerECWorkers(t, pluginSvc, map[string]string{
"worker-local": "10.0.0.1:9334",
"worker-remote": "10.0.0.2:9334",
})
jobs := []*plugin_pb.JobSpec{
replicaJob("local-1", "10.0.0.1:8080"),
replicaJob("local-2", "10.0.0.1:8080"),
replicaJob("homeless-1", "10.0.0.9:8080"),
replicaJob("homeless-2", "10.0.0.9:8080"),
}
policy := schedulerPolicy{
PerWorkerConcurrency: 1,
LocalQueuePin: 5,
OrphanRehomeGrace: 30 * time.Second,
}
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
if qs.queueLen("worker-local") != 2 || qs.queueLen("worker-remote") != 0 || qs.leftoverLen() != 2 {
t.Fatalf("homes local=%d remote=%d leftover=%d", qs.queueLen("worker-local"), qs.queueLen("worker-remote"), qs.leftoverLen())
}
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if item1 == nil || item1.job.JobId != "local-1" || target1 != "worker-local" {
t.Fatalf("first take want local-1 on home, got job=%v target=%s", item1, target1)
}
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
if err != nil {
t.Fatalf("reserve local: %v", err)
}
defer release()
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if item2 == nil || item2.job.JobId != "homeless-1" || target2 != "worker-remote" {
t.Fatalf("remote should take homeless, not steal pinned home, got job=%v target=%s", item2, target2)
}
if qs.queueLen("worker-local") != 1 {
t.Fatalf("remaining replica-local job must stay home, got %d", qs.queueLen("worker-local"))
}
if qs.leftoverLen() != 1 {
t.Fatalf("one homeless should remain, leftover=%d", qs.leftoverLen())
}
}
func TestIdleHomeWorkerTakesHomelessAfterEmptyHomeQueue(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
registerECWorkers(t, pluginSvc, map[string]string{
"worker-local": "10.0.0.1:9334",
})
jobs := []*plugin_pb.JobSpec{
replicaJob("home-1", "10.0.0.1:8080"),
replicaJob("homeless-1", "10.0.0.9:8080"),
}
policy := schedulerPolicy{
PerWorkerConcurrency: 1,
LocalQueuePin: 5,
OrphanRehomeGrace: 30 * time.Second,
}
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
if qs.queueLen("worker-local") != 1 || qs.leftoverLen() != 1 {
t.Fatalf("home=%d leftover=%d", qs.queueLen("worker-local"), qs.leftoverLen())
}
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if item1 == nil || item1.job.JobId != "home-1" || target1 != "worker-local" {
t.Fatalf("first take want home-1, got job=%v target=%s", item1, target1)
}
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if item2 == nil || item2.job.JobId != "homeless-1" || target2 != "worker-local" {
t.Fatalf("idle home with empty queue may take homeless, got job=%v target=%s", item2, target2)
}
if qs.leftoverLen() != 0 {
t.Fatalf("homeless should be gone, leftover=%d", qs.leftoverLen())
}
}
func TestHomelessWorkerTakesFIFOBeforeStealableHome(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
registerECWorkers(t, pluginSvc, map[string]string{
"worker-local": "10.0.0.1:9334",
"worker-remote": "10.0.0.2:9334",
})
jobs := make([]*plugin_pb.JobSpec, 0, 7)
for i := 0; i < 6; i++ {
jobs = append(jobs, replicaJob(fmt.Sprintf("j%d", i), "10.0.0.1:8080"))
}
jobs = append(jobs, replicaJob("homeless-1", "10.0.0.9:8080"))
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
if qs.queueLen("worker-local") != 6 || qs.leftoverLen() != 1 {
t.Fatalf("home=%d leftover=%d", qs.queueLen("worker-local"), qs.leftoverLen())
}
if qs.hasHome("worker-remote") {
t.Fatal("remote must have no home identity")
}
_, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if target != "worker-local" {
t.Fatalf("first take want local, got %s", target)
}
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
if err != nil {
t.Fatalf("reserve local: %v", err)
}
defer release()
item, takeTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if item == nil || takeTarget != "worker-remote" || item.job.JobId != "homeless-1" {
t.Fatalf("remote must take homeless FIFO before K-steal, got job=%v target=%s", item, takeTarget)
}
if qs.leftoverLen() != 0 {
t.Fatalf("homeless should be gone, leftover=%d", qs.leftoverLen())
}
if qs.queueLen("worker-local") != 5 {
t.Fatalf("stealable home must stay until FIFO empty, got %d", qs.queueLen("worker-local"))
}
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if stolen == nil || stealTarget != "worker-remote" || stolen.job.JobId == "homeless-1" {
t.Fatalf("after FIFO empty remote may steal >=K, got job=%v target=%s", stolen, stealTarget)
}
}
func TestHomeWorkerStealsBeforeHomelessAfterOwnQueueEmpty(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
registerECWorkers(t, pluginSvc, map[string]string{
"worker-local": "10.0.0.1:9334",
"worker-other": "10.0.0.2:9334",
})
jobs := make([]*plugin_pb.JobSpec, 0, 8)
for i := 0; i < 6; i++ {
jobs = append(jobs, replicaJob(fmt.Sprintf("j%d", i), "10.0.0.1:8080"))
}
jobs = append(jobs, replicaJob("other-1", "10.0.0.2:8080"))
jobs = append(jobs, replicaJob("homeless-1", "10.0.0.9:8080"))
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
if qs.queueLen("worker-local") != 6 || qs.queueLen("worker-other") != 1 || qs.leftoverLen() != 1 {
t.Fatalf("local=%d other=%d leftover=%d", qs.queueLen("worker-local"), qs.queueLen("worker-other"), qs.leftoverLen())
}
if !qs.hasHome("worker-other") {
t.Fatal("other must keep home identity")
}
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if item1 == nil || target1 != "worker-local" {
t.Fatalf("first take want local home, got job=%v target=%s", item1, target1)
}
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
if err != nil {
t.Fatalf("reserve local: %v", err)
}
defer release()
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if item2 == nil || item2.job.JobId != "other-1" || target2 != "worker-other" {
t.Fatalf("other should drain own home first, got job=%v target=%s", item2, target2)
}
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
if stolen == nil || stealTarget != "worker-other" || stolen.job.JobId == "homeless-1" {
t.Fatalf("home worker with empty own queue still steals before homeless, got job=%v target=%s", stolen, stealTarget)
}
if qs.leftoverLen() != 1 {
t.Fatalf("homeless should still wait, leftover=%d", qs.leftoverLen())
}
}
+8 -2
View File
@@ -82,7 +82,7 @@ func runPluginWorkerWithOptions(options pluginWorkerRunOptions) bool {
return false
}
handlers, err := buildPluginWorkerHandlers(options.JobTypes, dialOption, options.MaxExecute, options.WorkingDir)
handlers, err := buildPluginWorkerHandlers(options.JobTypes, dialOption, options.MaxExecute, options.WorkingDir, options.Address)
if err != nil {
glog.Errorf("Failed to build plugin worker handlers: %v", err)
return false
@@ -141,7 +141,7 @@ func resolvePluginWorkerID(explicitID string, workingDir string) (string, error)
// buildPluginWorkerHandlers resolves the comma-separated jobTypes string
// (which may contain category names like "all", "default", "heavy" and/or
// explicit job type names/aliases) into a deduplicated slice of JobHandlers.
func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxExecute int, workingDir string) ([]pluginworker.JobHandler, error) {
func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxExecute int, workingDir string, workerAddress ...string) ([]pluginworker.JobHandler, error) {
jobTypes = strings.TrimSpace(jobTypes)
if jobTypes == "" {
jobTypes = defaultPluginWorkerJobTypes
@@ -152,10 +152,16 @@ func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxE
return nil, err
}
addr := ""
if len(workerAddress) > 0 {
addr = strings.TrimSpace(workerAddress[0])
}
opts := pluginworker.HandlerBuildOptions{
GrpcDialOption: dialOption,
MaxExecute: maxExecute,
WorkingDir: workingDir,
WorkerAddress: addr,
}
handlers := make([]pluginworker.JobHandler, 0, len(factories))
+3
View File
@@ -37,6 +37,9 @@ type HandlerBuildOptions struct {
GrpcDialOption grpc.DialOption
MaxExecute int
WorkingDir string
// WorkerAddress is the advertised weed worker -address (host:port).
// It is not a volume-server URL; handlers match replicas by hostname.
WorkerAddress string
}
var (
+79
View File
@@ -0,0 +1,79 @@
package util
import (
"net"
"strings"
)
// HostKey is the hostname/IP of an "host:port" address (or the string itself
// when it has no port). Compared case-insensitively so worker Hello
// "10.87.0.3:33646" matches a volume replica on "10.87.0.3:8080".
func HostKey(addr string) string {
addr = strings.TrimSpace(addr)
if addr == "" {
return ""
}
host, _, err := net.SplitHostPort(addr)
if err != nil {
return strings.ToLower(addr)
}
return strings.ToLower(host)
}
// HostKeysFromAddresses returns unique HostKey values, skipping empties.
func HostKeysFromAddresses(addrs []string) []string {
seen := make(map[string]struct{}, len(addrs))
var out []string
for _, a := range addrs {
h := HostKey(a)
if h == "" {
continue
}
if _, ok := seen[h]; ok {
continue
}
seen[h] = struct{}{}
out = append(out, h)
}
return out
}
// AddressIsLocal reports whether addr's host is this process's machine
// (loopback or a configured interface address). Used so an EC worker prefers
// a replica on the same box after replica sync.
func AddressIsLocal(addr string) bool {
host := HostKey(addr)
if host == "" {
return false
}
if host == "localhost" || host == "127.0.0.1" || host == "::1" {
return true
}
ip := net.ParseIP(host)
if ip != nil && ip.IsLoopback() {
return true
}
ifaces, err := net.InterfaceAddrs()
if err != nil {
return false
}
for _, ia := range ifaces {
var got net.IP
switch v := ia.(type) {
case *net.IPNet:
got = v.IP
case *net.IPAddr:
got = v.IP
}
if got == nil {
continue
}
if strings.EqualFold(got.String(), host) {
return true
}
if ip != nil && got.Equal(ip) {
return true
}
}
return false
}
+24
View File
@@ -0,0 +1,24 @@
package util
import "testing"
func TestHostKeyStripsPort(t *testing.T) {
t.Parallel()
if got := HostKey("10.87.0.3:8080"); got != "10.87.0.3" {
t.Fatalf("HostKey replica: got %q", got)
}
if got := HostKey("10.87.0.3:33646"); got != "10.87.0.3" {
t.Fatalf("HostKey worker: got %q", got)
}
if HostKey("10.87.0.3:8080") != HostKey("10.87.0.3:33646") {
t.Fatal("worker and volume server on the same IP must share a HostKey")
}
}
func TestHostKeysFromAddressesDedups(t *testing.T) {
t.Parallel()
got := HostKeysFromAddresses([]string{"10.0.0.1:8080", "10.0.0.1:8081", "10.0.0.2:8080", ""})
if len(got) != 2 || got[0] != "10.0.0.1" || got[1] != "10.0.0.2" {
t.Fatalf("got %v", got)
}
}
@@ -21,6 +21,7 @@ import (
storagetypes "github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
"github.com/seaweedfs/seaweedfs/weed/storage/volume_replica"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
"github.com/seaweedfs/seaweedfs/weed/worker/types/base"
@@ -36,6 +37,9 @@ type ErasureCodingTask struct {
workDir string
progress float64
grpcDialOption grpc.DialOption
// workerAddress is weed worker -address (host:port). Not a volume URL;
// late-bind matches replica hostnames against HostKey(workerAddress).
workerAddress string
// EC parameters
dataShards int32
@@ -403,9 +407,75 @@ func (t *ErasureCodingTask) syncAndSelectSourceReplica() error {
glog.Infof("EC encode: using best replica %s as source for volume %d (was %s)", best.Url, t.volumeID, t.server)
t.server = best.Url
}
// Late-bind from after consistency: worker -address hostname, not Sources[0]
// and not AddressIsLocal (replica URLs are hostnames like full-02:8081).
workerAddr := strings.TrimSpace(t.workerAddress)
if workerAddr == "" {
workerAddr, _ = os.Hostname()
}
if local := preferWorkerHostReplicaIfComplete(t.grpcDialOption, needle.VolumeId(t.volumeID), locs, t.server, workerAddr); local != "" && local != t.server {
glog.Infof("EC encode: preferring worker-host replica %s as source for volume %d (was %s)", local, t.volumeID, t.server)
t.server = local
}
return nil
}
type replicaFileCount struct {
url string
fileCount uint64
err error
}
// preferWorkerHostReplicaIfComplete rebinds from to a same-hostname replica
// only when live FileCounts are equal. Diverged counts keep current (max).
func preferWorkerHostReplicaIfComplete(dial grpc.DialOption, vid needle.VolumeId, locs []wdclient.Location, current, workerAddress string) string {
statuses := volume_replica.GetReplicaStatuses(dial, vid, locs)
replicas := make([]replicaFileCount, 0, len(statuses))
for _, s := range statuses {
replicas = append(replicas, replicaFileCount{url: s.Location.Url, fileCount: s.FileCount, err: s.Error})
}
return lateBindSourceToWorkerHost(current, workerAddress, replicas)
}
// lateBindSourceToWorkerHost: equal FileCount → replica on worker -address host
// (any port). Diverge → keep current (already max FileCount). Worker address
// is not a volume URL.
func lateBindSourceToWorkerHost(current, workerAddress string, replicas []replicaFileCount) string {
workerHost := util.HostKey(workerAddress)
if workerHost == "" {
return current
}
var live []replicaFileCount
for _, r := range replicas {
if r.err != nil || r.url == "" {
continue
}
live = append(live, r)
}
if len(live) == 0 {
return current
}
maxCount := live[0].fileCount
allSame := true
for _, r := range live[1:] {
if r.fileCount != live[0].fileCount {
allSame = false
}
if r.fileCount > maxCount {
maxCount = r.fileCount
}
}
if !allSame {
return current
}
for _, r := range live {
if util.HostKey(r.url) == workerHost {
return r.url
}
}
return current
}
// rollbackReadonly is a best-effort restore of every replica markReplicasReadonly
// touched, used when the EC task fails before the originals are deleted. Logs but
// does not return errors; uses a fresh context since the caller's may be cancelled.
@@ -0,0 +1,31 @@
package erasure_coding
import "testing"
func TestLateBindSourceToWorkerHostEqualFileCountPrefersWorkerHost(t *testing.T) {
a := replicaFileCount{url: "full-01:8080", fileCount: 21}
b := replicaFileCount{url: "full-02:8081", fileCount: 21}
worker := "full-02:9334"
// Sources [A,B] — topology-first is A; worker on B must still copy B.
if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{a, b}); got != b.url {
t.Fatalf("Sources [A,B] worker B: got %s want %s", got, b.url)
}
// Invert Sources — still B.
if got := lateBindSourceToWorkerHost(b.url, worker, []replicaFileCount{b, a}); got != b.url {
t.Fatalf("Sources [B,A] worker B: got %s want %s", got, b.url)
}
}
func TestLateBindSourceToWorkerHostDivergeKeepsMaxFileCount(t *testing.T) {
a := replicaFileCount{url: "full-01:8080", fileCount: 40}
b := replicaFileCount{url: "full-02:8081", fileCount: 21}
worker := "full-02:9334"
if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{a, b}); got != a.url {
t.Fatalf("diverge [A,B]: got %s want max FileCount %s", got, a.url)
}
if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{b, a}); got != a.url {
t.Fatalf("diverge [B,A]: got %s want max FileCount %s", got, a.url)
}
}
@@ -26,7 +26,9 @@ func init() {
Category: pluginworker.CategoryHeavy,
Aliases: []string{"erasure-coding", "erasure.coding", "ec"},
Build: func(opts pluginworker.HandlerBuildOptions) (pluginworker.JobHandler, error) {
return NewErasureCodingHandler(opts.GrpcDialOption, opts.WorkingDir), nil
h := NewErasureCodingHandler(opts.GrpcDialOption, opts.WorkingDir)
h.workerAddress = strings.TrimSpace(opts.WorkerAddress)
return h, nil
},
})
}
@@ -39,6 +41,7 @@ type erasureCodingWorkerConfig struct {
type ErasureCodingHandler struct {
grpcDialOption grpc.DialOption
workingDir string
workerAddress string
}
func NewErasureCodingHandler(grpcDialOption grpc.DialOption, workingDir string) *ErasureCodingHandler {
@@ -519,6 +522,7 @@ func (h *ErasureCodingHandler) Execute(
params.Collection,
h.grpcDialOption,
)
task.workerAddress = h.workerAddress
execCtx, execCancel := context.WithCancel(ctx)
defer execCancel()
task.SetProgressCallback(func(progress float64, stage string) {
@@ -670,8 +674,15 @@ func buildErasureCodingProposal(
}
sourceNode := ""
if len(params.Sources) > 0 {
sourceNode = strings.TrimSpace(params.Sources[0].Node)
var replicaServers []string
for _, s := range params.Sources {
if s == nil || strings.TrimSpace(s.Node) == "" || len(s.ShardIds) > 0 {
continue
}
replicaServers = append(replicaServers, s.Node)
if sourceNode == "" {
sourceNode = strings.TrimSpace(s.Node)
}
}
summary := fmt.Sprintf("Erasure code volume %d", result.VolumeID)
@@ -702,6 +713,9 @@ func buildErasureCodingProposal(
"source_server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: sourceNode},
},
"replica_servers": {
Kind: &plugin_pb.ConfigValue_StringList{StringList: &plugin_pb.StringList{Values: replicaServers}},
},
"collection": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: result.Collection},
},