diff --git a/weed/admin/plugin/host_affinity.go b/weed/admin/plugin/host_affinity.go new file mode 100644 index 0000000000..ddff60ece0 --- /dev/null +++ b/weed/admin/plugin/host_affinity.go @@ -0,0 +1,66 @@ +package plugin + +import ( + "strings" + + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/worker_pb" + pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker" + "github.com/seaweedfs/seaweedfs/weed/util" + + "google.golang.org/protobuf/proto" +) + +// JobReplicaHosts are the volume-replica hosts a job will read from. +// Used so executor pick prefers a worker on one of those machines. +// Sources with ShardIds are EC-cleanup rows, not encode replicas. +func JobReplicaHosts(job *plugin_pb.JobSpec) []string { + if job == nil { + return nil + } + var addrs []string + addrs = append(addrs, pluginworker.ReadStringListConfig(job.Parameters, "replica_servers")...) + if src := strings.TrimSpace(pluginworker.ReadStringConfig(job.Parameters, "source_server", "")); src != "" { + addrs = append(addrs, src) + } + if payload := pluginworker.ReadBytesConfig(job.Parameters, "task_params_pb"); len(payload) > 0 { + params := &worker_pb.TaskParams{} + if err := proto.Unmarshal(payload, params); err == nil { + for _, s := range params.GetSources() { + if s == nil || s.Node == "" || len(s.ShardIds) > 0 { + continue + } + addrs = append(addrs, s.Node) + } + } + } + return util.HostKeysFromAddresses(addrs) +} + +// partitionByPreferredHost splits candidates into (host matches, others). +// No preferred hosts, or no matches, returns (nil, all) so callers keep +// the existing slots+round-robin path. +func partitionByPreferredHost(candidates []*WorkerSession, preferredHosts []string) (pref, rest []*WorkerSession) { + want := util.HostKeysFromAddresses(preferredHosts) + if len(want) == 0 || len(candidates) == 0 { + return nil, candidates + } + set := make(map[string]struct{}, len(want)) + for _, h := range want { + set[h] = struct{}{} + } + for _, c := range candidates { + if c == nil { + continue + } + if _, ok := set[util.HostKey(c.Address)]; ok { + pref = append(pref, c) + continue + } + rest = append(rest, c) + } + if len(pref) == 0 { + return nil, candidates + } + return pref, rest +} diff --git a/weed/admin/plugin/plugin.go b/weed/admin/plugin/plugin.go index 8ea35d415b..fda1a86586 100644 --- a/weed/admin/plugin/plugin.go +++ b/weed/admin/plugin/plugin.go @@ -615,7 +615,7 @@ func (r *Plugin) ExecuteJob( releaseGate := r.acquireDetectionExecutionGate(jobType, true) defer releaseGate() - executor, err := r.registry.PickExecutor(jobType) + executor, err := r.registry.PickExecutor(jobType, JobReplicaHosts(job)...) if err != nil { return nil, err } diff --git a/weed/admin/plugin/plugin_monitor.go b/weed/admin/plugin/plugin_monitor.go index 9a5327ca18..c8a58e298c 100644 --- a/weed/admin/plugin/plugin_monitor.go +++ b/weed/admin/plugin/plugin_monitor.go @@ -887,6 +887,37 @@ func (r *Plugin) trackExecutionQueued(job *plugin_pb.JobSpec) { }) } +// bindQueuedJobWorker records the home worker while the job stays pending/queued. +func (r *Plugin) bindQueuedJobWorker(job *plugin_pb.JobSpec, workerID string) { + if job == nil || strings.TrimSpace(job.JobId) == "" || strings.TrimSpace(workerID) == "" { + return + } + + now := time.Now().UTC() + r.jobsMu.Lock() + tracked := r.jobs[job.JobId] + if tracked == nil { + r.jobsMu.Unlock() + return + } + if isTerminalTrackedJobState(tracked.State) { + r.jobsMu.Unlock() + return + } + tracked.WorkerID = workerID + tracked.Message = "queued on worker " + workerID + tracked.UpdatedAt = timeToPtr(now) + snap := cloneTrackedJob(*tracked) + r.dirtyJobs = true + r.jobsMu.Unlock() + + r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) { + detail.WorkerID = snap.WorkerID + detail.Message = snap.Message + detail.UpdatedAt = snap.UpdatedAt + }) +} + func (r *Plugin) cancelQueuedJob(job *plugin_pb.JobSpec, cause error) { reason := "job canceled" if cause != nil { diff --git a/weed/admin/plugin/plugin_scheduler.go b/weed/admin/plugin/plugin_scheduler.go index 94a3aa3976..aa97e92737 100644 --- a/weed/admin/plugin/plugin_scheduler.go +++ b/weed/admin/plugin/plugin_scheduler.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sort" "strings" "sync" "sync/atomic" @@ -28,12 +29,18 @@ const ( defaultScheduledMaxResults int32 = 1000 defaultScheduledExecutionConcurrency = 1 defaultScheduledPerWorkerConcurrency = 1 - maxScheduledExecutionConcurrency = 128 - defaultScheduledRetryBackoff = 5 * time.Second - defaultClusterContextTimeout = 10 * time.Second - defaultWaitingBacklogFloor = 8 - defaultWaitingBacklogMultiplier = 4 - maxEstimatedRuntimeCap = 8 * time.Hour + // Hold a job on replica-host workers until their reservation load + // reaches this; 0 disables the pin and spills immediately. + defaultLocalQueuePin = 5 + // Wait this long after a home worker disappears before stealing + // its queue; 0 steals immediately. Covers a reconnect blink. + defaultOrphanRehomeGrace = 30 * time.Second + maxScheduledExecutionConcurrency = 128 + defaultScheduledRetryBackoff = 5 * time.Second + defaultClusterContextTimeout = 10 * time.Second + defaultWaitingBacklogFloor = 8 + defaultWaitingBacklogMultiplier = 4 + maxEstimatedRuntimeCap = 8 * time.Hour // How long started jobs may drain past the run window close. scheduledExecutionDrainGrace = 30 * time.Minute ) @@ -49,6 +56,12 @@ type schedulerPolicy struct { PerWorkerConcurrency int RetryLimit int ExecutorReserveBackoff time.Duration + // LocalQueuePin is K: do not spill off replica-host workers while + // min(load(pref)) < K. Zero is the old immediate-spill path. + LocalQueuePin int + // OrphanRehomeGrace is how long a missing home's queue is held + // before steal-without-K. Zero steals as soon as the worker is gone. + OrphanRehomeGrace time.Duration } // laneSchedulerLoop is the main scheduling goroutine for a single lane. @@ -466,6 +479,8 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err PerWorkerConcurrency: int(adminRuntime.PerWorkerExecutionConcurrency), RetryLimit: int(adminRuntime.RetryLimit), ExecutorReserveBackoff: 200 * time.Millisecond, + LocalQueuePin: defaultLocalQueuePin, + OrphanRehomeGrace: defaultOrphanRehomeGrace, } if policy.DetectionInterval < r.schedulerTick { @@ -496,6 +511,9 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err if policy.ExecutionTimeout < defaultScheduledExecutionTimeout { policy.ExecutionTimeout = defaultScheduledExecutionTimeout } + if policy.LocalQueuePin < 0 { + policy.LocalQueuePin = 0 + } return policy, true, nil } @@ -813,19 +831,22 @@ func (r *Plugin) dispatchScheduledProposals( // whole dispatch lets manual shell operations interleave between jobs. jobsRequireLock := LaneRequiresLock(JobTypeLane(jobType)) - jobQueue := make(chan *plugin_pb.JobSpec, len(proposals)) + jobs := make([]*plugin_pb.JobSpec, 0, len(proposals)) for index, proposal := range proposals { - job := buildScheduledJobSpec(jobType, proposal, index) - r.trackExecutionQueued(job) select { case <-r.shutdownCh: - close(jobQueue) - return 0, 0, 0 + for _, job := range jobs { + r.cancelQueuedJob(job, errSchedulerShutdown) + } + return 0, 0, len(jobs) default: - jobQueue <- job } + job := buildScheduledJobSpec(jobType, proposal, index) + r.trackExecutionQueued(job) + jobs = append(jobs, job) } - close(jobQueue) + + queues := r.assignJobsToWorkerQueues(jobType, jobs, policy) var wg sync.WaitGroup var statsMu sync.Mutex @@ -842,130 +863,110 @@ func (r *Plugin) dispatchScheduledProposals( wg.Add(1) go func() { defer wg.Done() - - jobLoop: - for job := range jobQueue { + for { select { case <-r.shutdownCh: return default: } - if ctx.Err() != nil { - r.cancelQueuedJob(job, ctx.Err()) - statsMu.Lock() - canceledCount++ - statsMu.Unlock() + return + } + if queues.isEmpty() { + return + } + + item, targetID := r.takeQueuedJob(queues, jobType, policy) + if item == nil { + if queues.isEmpty() { + return + } + if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) { + return + } + continue + } + if ctx.Err() != nil { + queues.pushFront(item) + return + } + + executor, release, reserveErr := r.reserveExecutorByID(jobType, policy, targetID) + if reserveErr != nil { + queues.pushFront(item) + if ctx.Err() != nil { + return + } + if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) { + return + } continue } - for { - select { - case <-r.shutdownCh: - return - default: + var releaseJobLock func() + if jobsRequireLock { + var lockErr error + releaseJobLock, lockErr = r.acquireAdminLock(fmt.Sprintf("plugin scheduler %s job %s", jobType, item.job.JobId)) + if lockErr != nil { + release() + statsMu.Lock() + errorCount++ + statsMu.Unlock() + r.appendActivity(JobActivity{ + JobID: item.job.JobId, + JobType: item.job.JobType, + Source: "admin_scheduler", + Message: fmt.Sprintf("scheduled execution lock failed: %v", lockErr), + Stage: "failed", + OccurredAt: timeToPtr(time.Now().UTC()), + }) + continue } if ctx.Err() != nil { - r.cancelQueuedJob(job, ctx.Err()) + releaseJobLock() + release() + queues.pushFront(item) + return + } + } + + err := r.executeScheduledJobWithExecutor(ctx, executor, item.job, clusterContext, policy) + if releaseJobLock != nil { + releaseJobLock() + } + release() + if errors.Is(err, errExecutorAtCapacity) { + queues.pushFront(item) + if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) { + return + } + continue + } + if err != nil { + if ctx.Err() != nil || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + r.cancelQueuedJob(item.job, err) statsMu.Lock() canceledCount++ statsMu.Unlock() - continue jobLoop - } - - executor, release, reserveErr := r.reserveScheduledExecutor(ctx, jobType, policy) - if reserveErr != nil { - if ctx.Err() != nil { - r.cancelQueuedJob(job, ctx.Err()) - statsMu.Lock() - canceledCount++ - statsMu.Unlock() - continue jobLoop - } - statsMu.Lock() - errorCount++ - statsMu.Unlock() - r.appendActivity(JobActivity{ - JobType: jobType, - Source: "admin_scheduler", - Message: fmt.Sprintf("scheduled execution reservation failed: %v", reserveErr), - Stage: "failed", - OccurredAt: timeToPtr(time.Now().UTC()), - }) - break - } - - var releaseJobLock func() - if jobsRequireLock { - var lockErr error - releaseJobLock, lockErr = r.acquireAdminLock(fmt.Sprintf("plugin scheduler %s job %s", jobType, job.JobId)) - if lockErr != nil { - release() - statsMu.Lock() - errorCount++ - statsMu.Unlock() - r.appendActivity(JobActivity{ - JobID: job.JobId, - JobType: job.JobType, - Source: "admin_scheduler", - Message: fmt.Sprintf("scheduled execution lock failed: %v", lockErr), - Stage: "failed", - OccurredAt: timeToPtr(time.Now().UTC()), - }) - break - } - // The lock may have been held by an operator for a - // while; re-check the execution window. - if ctx.Err() != nil { - releaseJobLock() - release() - r.cancelQueuedJob(job, ctx.Err()) - statsMu.Lock() - canceledCount++ - statsMu.Unlock() - continue jobLoop - } - } - - err := r.executeScheduledJobWithExecutor(ctx, executor, job, clusterContext, policy) - if releaseJobLock != nil { - releaseJobLock() - } - release() - if errors.Is(err, errExecutorAtCapacity) { - r.trackExecutionQueued(job) - if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) { - return - } continue } - if err != nil { - if ctx.Err() != nil || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { - r.cancelQueuedJob(job, err) - statsMu.Lock() - canceledCount++ - statsMu.Unlock() - continue jobLoop - } - statsMu.Lock() - errorCount++ - statsMu.Unlock() - r.appendActivity(JobActivity{ - JobID: job.JobId, - JobType: job.JobType, - Source: "admin_scheduler", - Message: fmt.Sprintf("scheduled execution failed: %v", err), - Stage: "failed", - OccurredAt: timeToPtr(time.Now().UTC()), - }) - break - } - statsMu.Lock() - successCount++ + errorCount++ statsMu.Unlock() - break + r.appendActivity(JobActivity{ + JobID: item.job.JobId, + JobType: item.job.JobType, + Source: "admin_scheduler", + Message: fmt.Sprintf("scheduled execution failed: %v", err), + Stage: "failed", + OccurredAt: timeToPtr(time.Now().UTC()), + }) + continue } + + statsMu.Lock() + successCount++ + statsMu.Unlock() } }() } @@ -976,8 +977,8 @@ func (r *Plugin) dispatchScheduledProposals( if drainErr == nil { drainErr = errSchedulerShutdown } - for job := range jobQueue { - r.cancelQueuedJob(job, drainErr) + for _, item := range queues.drain() { + r.cancelQueuedJob(item.job, drainErr) canceledCount++ } @@ -988,6 +989,7 @@ func (r *Plugin) reserveScheduledExecutor( ctx context.Context, jobType string, policy schedulerPolicy, + preferredHosts []string, ) (*WorkerSession, func(), error) { if ctx == nil { ctx = context.Background() @@ -1015,7 +1017,7 @@ func (r *Plugin) reserveScheduledExecutor( return nil, nil, fmt.Errorf("timed out waiting for executor capacity for %s", jobType) } - executors, err := r.registry.ListExecutors(jobType) + executors, err := r.registry.ListExecutors(jobType, preferredHosts...) if err != nil { if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) { if ctx.Err() != nil { @@ -1026,11 +1028,19 @@ func (r *Plugin) reserveScheduledExecutor( continue } - for _, executor := range executors { - release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy) - if !ok { - continue + pref, rest := partitionByPreferredHost(executors, preferredHosts) + lane := JobTypeLane(jobType) + loadOf := func(e *WorkerSession) int { return r.executorReservationLoad(e, lane) } + + if len(pref) > 0 { + sortExecutorsByLoad(pref, loadOf) + if executor, release, ok := r.tryReserveFirst(pref, jobType, policy); ok { + return executor, release, nil } + } + + sortExecutorsByLoad(rest, loadOf) + if executor, release, ok := r.tryReserveFirst(rest, jobType, policy); ok { return executor, release, nil } @@ -1051,6 +1061,75 @@ func (r *Plugin) tryReserveExecutorCapacity( return r.tryReserveExecutorCapacityForLane(executor, jobType, policy, JobTypeLane(jobType)) } +func (r *Plugin) tryReserveFirst(executors []*WorkerSession, jobType string, policy schedulerPolicy) (*WorkerSession, func(), bool) { + for _, executor := range executors { + release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy) + if ok { + return executor, release, true + } + } + return nil, nil, false +} + +func (r *Plugin) reserveExecutorByID(jobType string, policy schedulerPolicy, workerID string) (*WorkerSession, func(), error) { + workerID = strings.TrimSpace(workerID) + if workerID == "" { + return nil, nil, fmt.Errorf("executor id is empty") + } + executors, err := r.registry.ListExecutors(jobType) + if err != nil { + return nil, nil, err + } + for _, executor := range executors { + if executor == nil || executor.WorkerID != workerID { + continue + } + release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy) + if !ok { + return nil, nil, errExecutorAtCapacity + } + return executor, release, nil + } + return nil, nil, fmt.Errorf("executor %s is not available for %s", workerID, jobType) +} + +func (r *Plugin) executorReservationLoad(executor *WorkerSession, lane SchedulerLane) int { + if executor == nil { + return 0 + } + load := 0 + if executor.Heartbeat != nil && executor.Heartbeat.ExecutionSlotsUsed > 0 { + load = int(executor.Heartbeat.ExecutionSlotsUsed) + } + workerID := strings.TrimSpace(executor.WorkerID) + if ls := r.lanes[lane]; ls != nil { + ls.execMu.Lock() + load += ls.execRes[workerID] + ls.execMu.Unlock() + return load + } + r.schedulerExecMu.Lock() + load += r.schedulerExecReservations[workerID] + r.schedulerExecMu.Unlock() + return load +} + +func sortExecutorsByLoad(executors []*WorkerSession, load func(*WorkerSession) int) { + sort.SliceStable(executors, func(i, j int) bool { + if executors[i] == nil { + return false + } + if executors[j] == nil { + return true + } + li, lj := load(executors[i]), load(executors[j]) + if li != lj { + return li < lj + } + return executors[i].WorkerID < executors[j].WorkerID + }) +} + // tryReserveExecutorCapacityForLane reserves an execution slot on the // per-lane reservation pool so that lanes cannot starve each other. func (r *Plugin) tryReserveExecutorCapacityForLane( @@ -1577,6 +1656,8 @@ func (r *Plugin) DispatchProposals( ExecutionTimeout: defaultScheduledExecutionTimeout, RetryBackoff: defaultScheduledRetryBackoff, ExecutorReserveBackoff: 200 * time.Millisecond, + LocalQueuePin: defaultLocalQueuePin, + OrphanRehomeGrace: defaultOrphanRehomeGrace, } } diff --git a/weed/admin/plugin/plugin_scheduler_test.go b/weed/admin/plugin/plugin_scheduler_test.go index 2e9459bb95..b7aa3757ca 100644 --- a/weed/admin/plugin/plugin_scheduler_test.go +++ b/weed/admin/plugin/plugin_scheduler_test.go @@ -61,6 +61,12 @@ func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) { if policy.JobTypeMaxRuntime != 30*time.Minute { t.Fatalf("unexpected max runtime: got=%v", policy.JobTypeMaxRuntime) } + if policy.LocalQueuePin != defaultLocalQueuePin { + t.Fatalf("unexpected local queue pin: got=%d", policy.LocalQueuePin) + } + if policy.OrphanRehomeGrace != defaultOrphanRehomeGrace { + t.Fatalf("unexpected orphan rehome grace: got=%v", policy.OrphanRehomeGrace) + } } func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T) { @@ -138,13 +144,13 @@ func TestReserveScheduledExecutorRespectsPerWorkerLimit(t *testing.T) { ExecutorReserveBackoff: time.Millisecond, } - executor1, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy) + executor1, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil) if err != nil { t.Fatalf("reserve executor 1: %v", err) } defer release1() - executor2, release2, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy) + executor2, release2, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil) if err != nil { t.Fatalf("reserve executor 2: %v", err) } @@ -266,7 +272,7 @@ func TestReserveScheduledExecutorTimesOutWhenNoExecutor(t *testing.T) { start := time.Now() pluginSvc.Shutdown() - _, _, err = pluginSvc.reserveScheduledExecutor(context.Background(), "missing-job-type", policy) + _, _, err = pluginSvc.reserveScheduledExecutor(context.Background(), "missing-job-type", policy, nil) if err == nil { t.Fatalf("expected reservation shutdown error") } @@ -297,7 +303,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) { ExecutorReserveBackoff: 5 * time.Millisecond, } - _, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy) + _, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil) if err != nil { t.Fatalf("reserve executor 1: %v", err) } @@ -308,7 +314,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) { } secondReserveCh := make(chan reserveResult, 1) go func() { - _, release2, reserveErr := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy) + _, release2, reserveErr := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil) if release2 != nil { release2() } @@ -992,3 +998,50 @@ func TestLaneStatus_LifecycleNextDetectionSurvivesDefaultLanePrune(t *testing.T) status.NextDetectionAt, expected) } } + +func TestReserveWaterfillsPreferredHosts(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + ids := []string{"worker-a", "worker-b", "worker-c", "worker-d"} + for _, id := range ids { + pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: id, + Address: "10.0.0.1:9334", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 8}, + }, + }) + } + + policy := schedulerPolicy{ + PerWorkerConcurrency: 8, + ExecutorReserveBackoff: time.Millisecond, + LocalQueuePin: 5, + } + preferred := []string{"10.0.0.1:8080"} + + seen := map[string]int{} + var releases []func() + defer func() { + for _, release := range releases { + release() + } + }() + for i := 0; i < 4; i++ { + exec, release, err := pluginSvc.reserveScheduledExecutor(context.Background(), "erasure_coding", policy, preferred) + if err != nil { + t.Fatalf("reserve %d: %v", i, err) + } + releases = append(releases, release) + seen[exec.WorkerID]++ + } + if len(seen) != 4 { + t.Fatalf("want one job per worker, got %v", seen) + } +} diff --git a/weed/admin/plugin/registry.go b/weed/admin/plugin/registry.go index 46d51e79d9..2201e6a9c5 100644 --- a/weed/admin/plugin/registry.go +++ b/weed/admin/plugin/registry.go @@ -277,14 +277,24 @@ func (r *Registry) PickDetector(jobType string) (*WorkerSession, error) { } // PickExecutor picks one executor worker for a job type. -func (r *Registry) PickExecutor(jobType string) (*WorkerSession, error) { - return r.pickByKind(jobType, false) +// preferredHosts, when set, ranks workers whose Hello Address host matches a +// replica/source host ahead of others (same-machine EC pull). Empty means the +// previous slots+round-robin order. +func (r *Registry) PickExecutor(jobType string, preferredHosts ...string) (*WorkerSession, error) { + list, err := r.ListExecutors(jobType, preferredHosts...) + if err != nil { + return nil, err + } + return list[0], nil } // ListExecutors returns sorted executor candidates for one job type. // Ordering is by most available execution slots, then lexical worker ID. // The top tie group is rotated round-robin to prevent sticky assignment. -func (r *Registry) ListExecutors(jobType string) ([]*WorkerSession, error) { +// preferredHosts, when any worker matches, are listed first (then the rest); +// rotation applies only inside the preferred group so a local worker is not +// rotated away in favor of a remote one. +func (r *Registry) ListExecutors(jobType string, preferredHosts ...string) ([]*WorkerSession, error) { r.mu.Lock() defer r.mu.Unlock() @@ -293,8 +303,16 @@ func (r *Registry) ListExecutors(jobType string) ([]*WorkerSession, error) { return nil, fmt.Errorf("no executor worker available for job_type=%s", jobType) } - sortByKind(candidates, jobType, false) - r.rotateTopCandidatesLocked(candidates, jobType, false) + pref, rest := partitionByPreferredHost(candidates, preferredHosts) + if len(pref) > 0 { + sortByKind(pref, jobType, false) + r.rotateTopCandidatesLocked(pref, jobType, false) + sortByKind(rest, jobType, false) + candidates = append(pref, rest...) + } else { + sortByKind(candidates, jobType, false) + r.rotateTopCandidatesLocked(candidates, jobType, false) + } out := make([]*WorkerSession, 0, len(candidates)) for _, candidate := range candidates { diff --git a/weed/admin/plugin/registry_test.go b/weed/admin/plugin/registry_test.go index 1d331278cb..d023e3ec99 100644 --- a/weed/admin/plugin/registry_test.go +++ b/weed/admin/plugin/registry_test.go @@ -401,3 +401,70 @@ func TestRegistryHasCapableWorkerIgnoresNilCapability(t *testing.T) { t.Fatalf("expected HasCapableWorker to return false when capability has no detect or execute") } } + +func TestListExecutorsPrefersMatchingReplicaHost(t *testing.T) { + t.Parallel() + + r := NewRegistry() + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-remote", + Address: "10.87.0.4:33646", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 8}, + }, + }) + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-local", + Address: "10.87.0.3:33646", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 1}, + }, + }) + r.UpdateHeartbeat("worker-remote", &plugin_pb.WorkerHeartbeat{ + WorkerId: "worker-remote", ExecutionSlotsUsed: 0, ExecutionSlotsTotal: 8, + }) + r.UpdateHeartbeat("worker-local", &plugin_pb.WorkerHeartbeat{ + WorkerId: "worker-local", ExecutionSlotsUsed: 0, ExecutionSlotsTotal: 1, + }) + + // Without preference the emptier remote worker wins on slots. + plain, err := r.ListExecutors("erasure_coding") + if err != nil { + t.Fatalf("ListExecutors: %v", err) + } + if plain[0].WorkerID != "worker-remote" { + t.Fatalf("without preference want remote first, got %s", plain[0].WorkerID) + } + + // Replica on :8080 of the same host as worker-local. + pref, err := r.ListExecutors("erasure_coding", "10.87.0.3:8080") + if err != nil { + t.Fatalf("ListExecutors preferred: %v", err) + } + if pref[0].WorkerID != "worker-local" { + t.Fatalf("preferred host want worker-local first, got %s", pref[0].WorkerID) + } + if pref[1].WorkerID != "worker-remote" { + t.Fatalf("remote should still be fallback, got %s", pref[1].WorkerID) + } +} + +func TestJobReplicaHostsFromSourcesAndReplicaServers(t *testing.T) { + t.Parallel() + job := &plugin_pb.JobSpec{ + Parameters: map[string]*plugin_pb.ConfigValue{ + "source_server": { + Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "10.87.0.1:8080"}, + }, + "replica_servers": { + Kind: &plugin_pb.ConfigValue_StringList{ + StringList: &plugin_pb.StringList{Values: []string{"10.87.0.1:8080", "10.87.0.2:8081"}}, + }, + }, + }, + } + got := JobReplicaHosts(job) + if len(got) != 2 { + t.Fatalf("got %v", got) + } +} diff --git a/weed/admin/plugin/worker_job_queues.go b/weed/admin/plugin/worker_job_queues.go new file mode 100644 index 0000000000..1e036a816c --- /dev/null +++ b/weed/admin/plugin/worker_job_queues.go @@ -0,0 +1,317 @@ +package plugin + +import ( + "sync" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" +) + +// queuedJob is one scheduled spec parked on a home worker's admin-side queue. +type queuedJob struct { + job *plugin_pb.JobSpec + home string +} + +// workerJobQueues holds per-worker FIFOs for one dispatch batch. +// Home assignment is done once up front; run goroutines only pop/steal. +// leftover is the homeless FIFO: replica hostnames with no currently +// registered worker. Orphan grace does not apply here. +type workerJobQueues struct { + mu sync.Mutex + q map[string][]*queuedJob + leftover []*queuedJob + remaining int + missingSince map[string]time.Time + homes map[string]struct{} +} + +func newWorkerJobQueues() *workerJobQueues { + return &workerJobQueues{ + q: make(map[string][]*queuedJob), + missingSince: make(map[string]time.Time), + homes: make(map[string]struct{}), + } +} + +func (qs *workerJobQueues) hasHome(workerID string) bool { + if qs == nil || workerID == "" { + return false + } + _, ok := qs.homes[workerID] + return ok +} + +func (qs *workerJobQueues) push(home string, job *plugin_pb.JobSpec) { + if qs == nil || job == nil { + return + } + item := &queuedJob{job: job, home: home} + qs.mu.Lock() + defer qs.mu.Unlock() + qs.pushLocked(item, false) +} + +func (qs *workerJobQueues) pushFront(item *queuedJob) { + if qs == nil || item == nil { + return + } + qs.mu.Lock() + defer qs.mu.Unlock() + qs.pushLocked(item, true) +} + +func (qs *workerJobQueues) pushLocked(item *queuedJob, front bool) { + qs.remaining++ + if item.home == "" { + if front { + qs.leftover = append([]*queuedJob{item}, qs.leftover...) + return + } + qs.leftover = append(qs.leftover, item) + return + } + if front { + qs.q[item.home] = append([]*queuedJob{item}, qs.q[item.home]...) + return + } + qs.q[item.home] = append(qs.q[item.home], item) +} + +func (qs *workerJobQueues) isEmpty() bool { + if qs == nil { + return true + } + qs.mu.Lock() + defer qs.mu.Unlock() + return qs.remaining == 0 +} + +func (qs *workerJobQueues) queueLen(workerID string) int { + if qs == nil { + return 0 + } + qs.mu.Lock() + defer qs.mu.Unlock() + return len(qs.q[workerID]) +} + +func (qs *workerJobQueues) leftoverLen() int { + if qs == nil { + return 0 + } + qs.mu.Lock() + defer qs.mu.Unlock() + return len(qs.leftover) +} + +func (qs *workerJobQueues) drain() []*queuedJob { + if qs == nil { + return nil + } + qs.mu.Lock() + defer qs.mu.Unlock() + out := make([]*queuedJob, 0, qs.remaining) + for id, qq := range qs.q { + out = append(out, qq...) + qs.q[id] = nil + } + out = append(out, qs.leftover...) + qs.leftover = nil + qs.remaining = 0 + return out +} + +func popFront(qq []*queuedJob) (*queuedJob, []*queuedJob) { + item := qq[0] + if len(qq) == 1 { + return item, nil + } + return item, qq[1:] +} + +// assignJobsToWorkerQueues parks each job on the least-loaded replica-host +// worker. A home exists only if a currently registered worker's -address +// hostname matches a replica host. Otherwise the job is homeless: global +// leftover FIFO, no fake home, no K, no orphan grace. Single-threaded. +func (r *Plugin) assignJobsToWorkerQueues( + jobType string, + jobs []*plugin_pb.JobSpec, + _ schedulerPolicy, +) *workerJobQueues { + qs := newWorkerJobQueues() + executors, err := r.registry.ListExecutors(jobType) + lane := JobTypeLane(jobType) + queued := make(map[string]int, len(executors)) + + for _, job := range jobs { + if job == nil { + continue + } + if err != nil || len(executors) == 0 { + qs.push("", job) + continue + } + pref, _ := partitionByPreferredHost(executors, JobReplicaHosts(job)) + if len(pref) == 0 { + // Never registered for that hostname: homeless, not an orphan. + qs.push("", job) + continue + } + for _, e := range pref { + if e != nil && e.WorkerID != "" { + qs.homes[e.WorkerID] = struct{}{} + } + } + sortExecutorsByLoad(pref, func(e *WorkerSession) int { + return r.executorReservationLoad(e, lane) + queued[e.WorkerID] + }) + home := pref[0].WorkerID + qs.push(home, job) + queued[home]++ + r.bindQueuedJobWorker(job, home) + } + return qs +} + +// takeQueuedJob pops a job for a free worker. +// Home worker (hostname matched a replica / has a home-queue identity): +// 1. own home queue +// 2. steal from another home if len>=K, or orphan past OrphanRehomeGrace +// 3. homeless leftover FIFO +// No-home worker (no matching replica, no home-queue identity): +// 1. homeless leftover FIFO +// 2. only if leftover empty: steal (K / orphan), same rules +func (r *Plugin) takeQueuedJob( + qs *workerJobQueues, + jobType string, + policy schedulerPolicy, +) (item *queuedJob, targetID string) { + if qs == nil { + return nil, "" + } + executors, err := r.registry.ListExecutors(jobType) + if err != nil || len(executors) == 0 { + return nil, "" + } + lane := JobTypeLane(jobType) + pin := policy.LocalQueuePin + defer func() { + if item != nil && targetID != "" { + r.bindQueuedJobWorker(item.job, targetID) + } + }() + + qs.mu.Lock() + defer qs.mu.Unlock() + if qs.remaining == 0 { + return nil, "" + } + + var bestHome *WorkerSession + bestLoad := 0 + for _, exec := range executors { + if len(qs.q[exec.WorkerID]) == 0 { + continue + } + limit := schedulerWorkerExecutionLimit(exec, jobType, policy) + if limit <= 0 { + continue + } + load := r.executorReservationLoad(exec, lane) + if load >= limit { + continue + } + if bestHome == nil || load < bestLoad || (load == bestLoad && exec.WorkerID < bestHome.WorkerID) { + bestHome = exec + bestLoad = load + } + } + if bestHome != nil { + id := bestHome.WorkerID + item, qs.q[id] = popFront(qs.q[id]) + qs.remaining-- + return item, id + } + + var thief *WorkerSession + for _, exec := range executors { + limit := schedulerWorkerExecutionLimit(exec, jobType, policy) + if limit <= 0 { + continue + } + if r.executorReservationLoad(exec, lane) >= limit { + continue + } + if thief == nil || exec.WorkerID < thief.WorkerID { + thief = exec + } + } + if thief == nil { + return nil, "" + } + + takeLeftover := func() (*queuedJob, string) { + if len(qs.leftover) == 0 { + return nil, "" + } + item, qs.leftover = popFront(qs.leftover) + qs.remaining-- + return item, thief.WorkerID + } + if !qs.hasHome(thief.WorkerID) { + if item, id := takeLeftover(); item != nil { + return item, id + } + } + + live := make(map[string]struct{}, len(executors)) + for _, exec := range executors { + live[exec.WorkerID] = struct{}{} + delete(qs.missingSince, exec.WorkerID) + } + + now := time.Now() + grace := policy.OrphanRehomeGrace + victimID := "" + victimLen := 0 + victimOrphan := false + for id, qq := range qs.q { + if len(qq) == 0 { + delete(qs.missingSince, id) + continue + } + _, isLive := live[id] + orphan := !isLive + if isLive { + if pin > 0 && len(qq) < pin { + continue + } + } else { + if _, ok := qs.missingSince[id]; !ok { + qs.missingSince[id] = now + } + if grace > 0 && now.Sub(qs.missingSince[id]) < grace { + continue + } + } + better := victimID == "" + if !better && orphan != victimOrphan { + better = orphan + } + if !better && orphan == victimOrphan { + better = len(qq) > victimLen || (len(qq) == victimLen && id < victimID) + } + if better { + victimID = id + victimLen = len(qq) + victimOrphan = orphan + } + } + if victimID != "" { + item, qs.q[victimID] = popFront(qs.q[victimID]) + qs.remaining-- + return item, thief.WorkerID + } + return takeLeftover() +} diff --git a/weed/admin/plugin/worker_job_queues_test.go b/weed/admin/plugin/worker_job_queues_test.go new file mode 100644 index 0000000000..662ac4e7b7 --- /dev/null +++ b/weed/admin/plugin/worker_job_queues_test.go @@ -0,0 +1,600 @@ +package plugin + +import ( + "fmt" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" +) + +func replicaJob(id, source string) *plugin_pb.JobSpec { + return &plugin_pb.JobSpec{ + JobId: id, + JobType: "erasure_coding", + Summary: id, + DedupeKey: id, + Parameters: map[string]*plugin_pb.ConfigValue{ + "source_server": { + Kind: &plugin_pb.ConfigValue_StringValue{StringValue: source}, + }, + }, + } +} + +func registerECWorkers(t *testing.T, pluginSvc *Plugin, workers map[string]string) { + t.Helper() + for id, addr := range workers { + pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: id, + Address: addr, + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 8}, + }, + }) + } +} + +func TestAssignJobsWaterfillsPreferredWorkers(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + registerECWorkers(t, pluginSvc, map[string]string{ + "worker-a": "10.0.0.1:9334", + "worker-b": "10.0.0.2:9334", + "worker-c": "10.0.0.3:9334", + "worker-d": "10.0.0.4:9334", + }) + + jobs := []*plugin_pb.JobSpec{ + replicaJob("j1", "10.0.0.1:8080"), + replicaJob("j2", "10.0.0.1:8080"), + replicaJob("j3", "10.0.0.1:8080"), + replicaJob("j4", "10.0.0.1:8080"), + } + // All four replica IPs listed so every worker is preferred. + for _, job := range jobs { + job.Parameters["replica_servers"] = &plugin_pb.ConfigValue{ + Kind: &plugin_pb.ConfigValue_StringList{StringList: &plugin_pb.StringList{ + Values: []string{"10.0.0.1:8080", "10.0.0.2:8080", "10.0.0.3:8080", "10.0.0.4:8080"}, + }}, + } + } + + qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, schedulerPolicy{LocalQueuePin: 5}) + if qs.queueLen("worker-a")+qs.queueLen("worker-b")+qs.queueLen("worker-c")+qs.queueLen("worker-d") != 4 { + t.Fatalf("expected 4 assigned jobs") + } + for _, id := range []string{"worker-a", "worker-b", "worker-c", "worker-d"} { + if qs.queueLen(id) != 1 { + t.Fatalf("want 1 job on %s, got %d", id, qs.queueLen(id)) + } + } +} + +func TestAssignAndTakeDoesNotStarveOtherHomes(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + registerECWorkers(t, pluginSvc, map[string]string{ + "worker-local": "10.0.0.1:9334", + "worker-remote": "10.0.0.2:9334", + }) + + jobs := []*plugin_pb.JobSpec{ + replicaJob("local-1", "10.0.0.1:8080"), + replicaJob("local-2", "10.0.0.1:8080"), + replicaJob("remote-1", "10.0.0.2:8080"), + } + policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5} + qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy) + if qs.queueLen("worker-local") != 2 || qs.queueLen("worker-remote") != 1 { + t.Fatalf("homes local=%d remote=%d", qs.queueLen("worker-local"), qs.queueLen("worker-remote")) + } + + _, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + _, release1, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, target1) + if err != nil { + t.Fatalf("reserve 1: %v", err) + } + defer release1() + + item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if item2 == nil || target2 != "worker-remote" { + t.Fatalf("second take should start remote home, got job=%v target=%s", item2, target2) + } +} + +func TestTakeStealsOnlyWhenHomeQueueReachesK(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + registerECWorkers(t, pluginSvc, map[string]string{ + "worker-local": "10.0.0.1:9334", + "worker-remote": "10.0.0.2:9334", + }) + + jobs := make([]*plugin_pb.JobSpec, 0, 6) + for i := 0; i < 6; i++ { + jobs = append(jobs, replicaJob(fmt.Sprintf("j%d", i), "10.0.0.1:8080")) + } + policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5} + qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy) + if qs.queueLen("worker-local") != 6 { + t.Fatalf("all jobs should home on local, got %d", qs.queueLen("worker-local")) + } + + _, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if target != "worker-local" { + t.Fatalf("first take want local, got %s", target) + } + _, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local") + if err != nil { + t.Fatalf("reserve local: %v", err) + } + defer release() + + // 5 still queued on local (>=K) and local has no slot → steal to remote. + stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if stolen == nil || stealTarget != "worker-remote" { + t.Fatalf("expected steal to remote, got job=%v target=%s", stolen, stealTarget) + } + if qs.queueLen("worker-local") != 4 { + t.Fatalf("after steal want 4 left on local, got %d", qs.queueLen("worker-local")) + } + + // 4 < K: do not steal again. + if item, tgt := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy); item != nil { + t.Fatalf("pin should hold remaining jobs, got target=%s", tgt) + } +} + +func TestTakeStealsImmediatelyWhenPinDisabled(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + registerECWorkers(t, pluginSvc, map[string]string{ + "worker-local": "10.0.0.1:9334", + "worker-remote": "10.0.0.2:9334", + }) + + jobs := []*plugin_pb.JobSpec{ + replicaJob("a", "10.0.0.1:8080"), + replicaJob("b", "10.0.0.1:8080"), + } + policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 0} + qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy) + + _, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if target != "worker-local" { + t.Fatalf("first take want local, got %s", target) + } + _, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local") + if err != nil { + t.Fatalf("reserve local: %v", err) + } + defer release() + + stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if stolen == nil || stealTarget != "worker-remote" { + t.Fatalf("K=0 should steal immediately, got job=%v target=%s", stolen, stealTarget) + } +} + +func TestTakeStealsOrphanedHomeWithoutK(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + registerECWorkers(t, pluginSvc, map[string]string{ + "worker-local": "10.0.0.1:9334", + "worker-remote": "10.0.0.2:9334", + }) + + jobs := []*plugin_pb.JobSpec{ + replicaJob("o1", "10.0.0.1:8080"), + replicaJob("o2", "10.0.0.1:8080"), + } + policy := schedulerPolicy{ + PerWorkerConcurrency: 1, + LocalQueuePin: 5, + OrphanRehomeGrace: 30 * time.Second, + } + qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy) + if qs.queueLen("worker-local") != 2 { + t.Fatalf("want 2 on local, got %d", qs.queueLen("worker-local")) + } + + pluginSvc.registry.Remove("worker-local") + + if item, tgt := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy); item != nil { + t.Fatalf("within grace should hold orphan queue, got target=%s", tgt) + } + if qs.queueLen("worker-local") != 2 { + t.Fatalf("held queue changed: %d", qs.queueLen("worker-local")) + } + + qs.mu.Lock() + qs.missingSince["worker-local"] = time.Now().Add(-time.Minute) + qs.mu.Unlock() + + stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if stolen == nil || stealTarget != "worker-remote" { + t.Fatalf("after grace should steal without K, got job=%v target=%s", stolen, stealTarget) + } + if qs.queueLen("worker-local") != 1 { + t.Fatalf("want 1 orphan left, got %d", qs.queueLen("worker-local")) + } + + stolen2, stealTarget2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if stolen2 == nil || stealTarget2 != "worker-remote" { + t.Fatalf("second orphan should also move, got job=%v target=%s", stolen2, stealTarget2) + } +} + +func TestTakeKeepsOrphanWhenHomeReconnectsDuringGrace(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + registerECWorkers(t, pluginSvc, map[string]string{ + "worker-local": "10.0.0.1:9334", + "worker-remote": "10.0.0.2:9334", + }) + + jobs := []*plugin_pb.JobSpec{replicaJob("blink", "10.0.0.1:8080")} + policy := schedulerPolicy{ + PerWorkerConcurrency: 1, + LocalQueuePin: 5, + OrphanRehomeGrace: 30 * time.Second, + } + qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy) + pluginSvc.registry.Remove("worker-local") + if item, _ := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy); item != nil { + t.Fatal("expected hold during grace") + } + + registerECWorkers(t, pluginSvc, map[string]string{"worker-local": "10.0.0.1:9334"}) + item, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if item == nil || target != "worker-local" { + t.Fatalf("reconnected home should run its queue, got job=%v target=%s", item, target) + } +} + +func TestTakeStealsOrphanImmediatelyWhenGraceDisabled(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + registerECWorkers(t, pluginSvc, map[string]string{ + "worker-local": "10.0.0.1:9334", + "worker-remote": "10.0.0.2:9334", + }) + + jobs := []*plugin_pb.JobSpec{replicaJob("o1", "10.0.0.1:8080")} + policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5, OrphanRehomeGrace: 0} + qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy) + pluginSvc.registry.Remove("worker-local") + + stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if stolen == nil || stealTarget != "worker-remote" { + t.Fatalf("grace=0 should steal immediately, got job=%v target=%s", stolen, stealTarget) + } +} + +func TestBindQueuedJobWorkerKeepsPendingState(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + job := replicaJob("bind-1", "10.0.0.1:8080") + pluginSvc.trackExecutionQueued(job) + pluginSvc.bindQueuedJobWorker(job, "worker-local") + + pluginSvc.jobsMu.Lock() + tracked := pluginSvc.jobs[job.JobId] + pluginSvc.jobsMu.Unlock() + if tracked == nil { + t.Fatal("missing tracked job") + } + if tracked.State != "job_state_pending" { + t.Fatalf("state=%s", tracked.State) + } + if tracked.Stage != "queued" { + t.Fatalf("stage=%s", tracked.Stage) + } + if tracked.WorkerID != "worker-local" { + t.Fatalf("worker=%s", tracked.WorkerID) + } +} + +func TestHomelessAllRemoteWorkersUseFIFOWithoutPinOrGrace(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + // Workers never registered on the replica hostname — homeless, not orphan. + registerECWorkers(t, pluginSvc, map[string]string{ + "worker-a": "10.0.0.5:9334", + "worker-b": "10.0.0.6:9334", + }) + + jobs := []*plugin_pb.JobSpec{ + replicaJob("h1", "10.0.0.1:8080"), + replicaJob("h2", "10.0.0.1:8080"), + replicaJob("h3", "10.0.0.1:8080"), + } + policy := schedulerPolicy{ + PerWorkerConcurrency: 1, + LocalQueuePin: 5, + OrphanRehomeGrace: 30 * time.Second, + } + qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy) + if qs.queueLen("worker-a") != 0 || qs.queueLen("worker-b") != 0 { + t.Fatalf("homeless jobs must not get a fake home, a=%d b=%d", qs.queueLen("worker-a"), qs.queueLen("worker-b")) + } + if qs.leftoverLen() != 3 { + t.Fatalf("want 3 homeless FIFO, got %d", qs.leftoverLen()) + } + + item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if item1 == nil || item1.job.JobId != "h1" { + t.Fatalf("FIFO first want h1, got %v", item1) + } + _, release1, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, target1) + if err != nil { + t.Fatalf("reserve 1: %v", err) + } + defer release1() + + item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if item2 == nil || item2.job.JobId != "h2" { + t.Fatalf("FIFO second want h2 immediately, got %v", item2) + } + if target2 == "" || target2 == target1 { + t.Fatalf("second take should hit the other idle worker, t1=%s t2=%s", target1, target2) + } + if qs.leftoverLen() != 1 { + t.Fatalf("want 1 homeless left, got %d", qs.leftoverLen()) + } +} + +func TestMixedReplicaLocalJobsStayHomeBeforeHomeless(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + registerECWorkers(t, pluginSvc, map[string]string{ + "worker-local": "10.0.0.1:9334", + "worker-remote": "10.0.0.2:9334", + }) + + jobs := []*plugin_pb.JobSpec{ + replicaJob("local-1", "10.0.0.1:8080"), + replicaJob("local-2", "10.0.0.1:8080"), + replicaJob("homeless-1", "10.0.0.9:8080"), + replicaJob("homeless-2", "10.0.0.9:8080"), + } + policy := schedulerPolicy{ + PerWorkerConcurrency: 1, + LocalQueuePin: 5, + OrphanRehomeGrace: 30 * time.Second, + } + qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy) + if qs.queueLen("worker-local") != 2 || qs.queueLen("worker-remote") != 0 || qs.leftoverLen() != 2 { + t.Fatalf("homes local=%d remote=%d leftover=%d", qs.queueLen("worker-local"), qs.queueLen("worker-remote"), qs.leftoverLen()) + } + + item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if item1 == nil || item1.job.JobId != "local-1" || target1 != "worker-local" { + t.Fatalf("first take want local-1 on home, got job=%v target=%s", item1, target1) + } + _, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local") + if err != nil { + t.Fatalf("reserve local: %v", err) + } + defer release() + + item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if item2 == nil || item2.job.JobId != "homeless-1" || target2 != "worker-remote" { + t.Fatalf("remote should take homeless, not steal pinned home, got job=%v target=%s", item2, target2) + } + if qs.queueLen("worker-local") != 1 { + t.Fatalf("remaining replica-local job must stay home, got %d", qs.queueLen("worker-local")) + } + if qs.leftoverLen() != 1 { + t.Fatalf("one homeless should remain, leftover=%d", qs.leftoverLen()) + } +} + +func TestIdleHomeWorkerTakesHomelessAfterEmptyHomeQueue(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + registerECWorkers(t, pluginSvc, map[string]string{ + "worker-local": "10.0.0.1:9334", + }) + + jobs := []*plugin_pb.JobSpec{ + replicaJob("home-1", "10.0.0.1:8080"), + replicaJob("homeless-1", "10.0.0.9:8080"), + } + policy := schedulerPolicy{ + PerWorkerConcurrency: 1, + LocalQueuePin: 5, + OrphanRehomeGrace: 30 * time.Second, + } + qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy) + if qs.queueLen("worker-local") != 1 || qs.leftoverLen() != 1 { + t.Fatalf("home=%d leftover=%d", qs.queueLen("worker-local"), qs.leftoverLen()) + } + + item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if item1 == nil || item1.job.JobId != "home-1" || target1 != "worker-local" { + t.Fatalf("first take want home-1, got job=%v target=%s", item1, target1) + } + item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if item2 == nil || item2.job.JobId != "homeless-1" || target2 != "worker-local" { + t.Fatalf("idle home with empty queue may take homeless, got job=%v target=%s", item2, target2) + } + if qs.leftoverLen() != 0 { + t.Fatalf("homeless should be gone, leftover=%d", qs.leftoverLen()) + } +} + +func TestHomelessWorkerTakesFIFOBeforeStealableHome(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + registerECWorkers(t, pluginSvc, map[string]string{ + "worker-local": "10.0.0.1:9334", + "worker-remote": "10.0.0.2:9334", + }) + + jobs := make([]*plugin_pb.JobSpec, 0, 7) + for i := 0; i < 6; i++ { + jobs = append(jobs, replicaJob(fmt.Sprintf("j%d", i), "10.0.0.1:8080")) + } + jobs = append(jobs, replicaJob("homeless-1", "10.0.0.9:8080")) + policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5} + qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy) + if qs.queueLen("worker-local") != 6 || qs.leftoverLen() != 1 { + t.Fatalf("home=%d leftover=%d", qs.queueLen("worker-local"), qs.leftoverLen()) + } + if qs.hasHome("worker-remote") { + t.Fatal("remote must have no home identity") + } + + _, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if target != "worker-local" { + t.Fatalf("first take want local, got %s", target) + } + _, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local") + if err != nil { + t.Fatalf("reserve local: %v", err) + } + defer release() + + item, takeTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if item == nil || takeTarget != "worker-remote" || item.job.JobId != "homeless-1" { + t.Fatalf("remote must take homeless FIFO before K-steal, got job=%v target=%s", item, takeTarget) + } + if qs.leftoverLen() != 0 { + t.Fatalf("homeless should be gone, leftover=%d", qs.leftoverLen()) + } + if qs.queueLen("worker-local") != 5 { + t.Fatalf("stealable home must stay until FIFO empty, got %d", qs.queueLen("worker-local")) + } + + stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if stolen == nil || stealTarget != "worker-remote" || stolen.job.JobId == "homeless-1" { + t.Fatalf("after FIFO empty remote may steal >=K, got job=%v target=%s", stolen, stealTarget) + } +} + +func TestHomeWorkerStealsBeforeHomelessAfterOwnQueueEmpty(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + registerECWorkers(t, pluginSvc, map[string]string{ + "worker-local": "10.0.0.1:9334", + "worker-other": "10.0.0.2:9334", + }) + + jobs := make([]*plugin_pb.JobSpec, 0, 8) + for i := 0; i < 6; i++ { + jobs = append(jobs, replicaJob(fmt.Sprintf("j%d", i), "10.0.0.1:8080")) + } + jobs = append(jobs, replicaJob("other-1", "10.0.0.2:8080")) + jobs = append(jobs, replicaJob("homeless-1", "10.0.0.9:8080")) + policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5} + qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy) + if qs.queueLen("worker-local") != 6 || qs.queueLen("worker-other") != 1 || qs.leftoverLen() != 1 { + t.Fatalf("local=%d other=%d leftover=%d", qs.queueLen("worker-local"), qs.queueLen("worker-other"), qs.leftoverLen()) + } + if !qs.hasHome("worker-other") { + t.Fatal("other must keep home identity") + } + + item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if item1 == nil || target1 != "worker-local" { + t.Fatalf("first take want local home, got job=%v target=%s", item1, target1) + } + _, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local") + if err != nil { + t.Fatalf("reserve local: %v", err) + } + defer release() + + item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if item2 == nil || item2.job.JobId != "other-1" || target2 != "worker-other" { + t.Fatalf("other should drain own home first, got job=%v target=%s", item2, target2) + } + + stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy) + if stolen == nil || stealTarget != "worker-other" || stolen.job.JobId == "homeless-1" { + t.Fatalf("home worker with empty own queue still steals before homeless, got job=%v target=%s", stolen, stealTarget) + } + if qs.leftoverLen() != 1 { + t.Fatalf("homeless should still wait, leftover=%d", qs.leftoverLen()) + } +} diff --git a/weed/command/worker_runtime.go b/weed/command/worker_runtime.go index 894edb5526..a60aa269c0 100644 --- a/weed/command/worker_runtime.go +++ b/weed/command/worker_runtime.go @@ -82,7 +82,7 @@ func runPluginWorkerWithOptions(options pluginWorkerRunOptions) bool { return false } - handlers, err := buildPluginWorkerHandlers(options.JobTypes, dialOption, options.MaxExecute, options.WorkingDir) + handlers, err := buildPluginWorkerHandlers(options.JobTypes, dialOption, options.MaxExecute, options.WorkingDir, options.Address) if err != nil { glog.Errorf("Failed to build plugin worker handlers: %v", err) return false @@ -141,7 +141,7 @@ func resolvePluginWorkerID(explicitID string, workingDir string) (string, error) // buildPluginWorkerHandlers resolves the comma-separated jobTypes string // (which may contain category names like "all", "default", "heavy" and/or // explicit job type names/aliases) into a deduplicated slice of JobHandlers. -func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxExecute int, workingDir string) ([]pluginworker.JobHandler, error) { +func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxExecute int, workingDir string, workerAddress ...string) ([]pluginworker.JobHandler, error) { jobTypes = strings.TrimSpace(jobTypes) if jobTypes == "" { jobTypes = defaultPluginWorkerJobTypes @@ -152,10 +152,16 @@ func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxE return nil, err } + addr := "" + if len(workerAddress) > 0 { + addr = strings.TrimSpace(workerAddress[0]) + } + opts := pluginworker.HandlerBuildOptions{ GrpcDialOption: dialOption, MaxExecute: maxExecute, WorkingDir: workingDir, + WorkerAddress: addr, } handlers := make([]pluginworker.JobHandler, 0, len(factories)) diff --git a/weed/plugin/worker/handler_registry.go b/weed/plugin/worker/handler_registry.go index 029382c060..cddd04b3c6 100644 --- a/weed/plugin/worker/handler_registry.go +++ b/weed/plugin/worker/handler_registry.go @@ -37,6 +37,9 @@ type HandlerBuildOptions struct { GrpcDialOption grpc.DialOption MaxExecute int WorkingDir string + // WorkerAddress is the advertised weed worker -address (host:port). + // It is not a volume-server URL; handlers match replicas by hostname. + WorkerAddress string } var ( diff --git a/weed/util/net_host.go b/weed/util/net_host.go new file mode 100644 index 0000000000..aa6fd385ca --- /dev/null +++ b/weed/util/net_host.go @@ -0,0 +1,79 @@ +package util + +import ( + "net" + "strings" +) + +// HostKey is the hostname/IP of an "host:port" address (or the string itself +// when it has no port). Compared case-insensitively so worker Hello +// "10.87.0.3:33646" matches a volume replica on "10.87.0.3:8080". +func HostKey(addr string) string { + addr = strings.TrimSpace(addr) + if addr == "" { + return "" + } + host, _, err := net.SplitHostPort(addr) + if err != nil { + return strings.ToLower(addr) + } + return strings.ToLower(host) +} + +// HostKeysFromAddresses returns unique HostKey values, skipping empties. +func HostKeysFromAddresses(addrs []string) []string { + seen := make(map[string]struct{}, len(addrs)) + var out []string + for _, a := range addrs { + h := HostKey(a) + if h == "" { + continue + } + if _, ok := seen[h]; ok { + continue + } + seen[h] = struct{}{} + out = append(out, h) + } + return out +} + +// AddressIsLocal reports whether addr's host is this process's machine +// (loopback or a configured interface address). Used so an EC worker prefers +// a replica on the same box after replica sync. +func AddressIsLocal(addr string) bool { + host := HostKey(addr) + if host == "" { + return false + } + if host == "localhost" || host == "127.0.0.1" || host == "::1" { + return true + } + ip := net.ParseIP(host) + if ip != nil && ip.IsLoopback() { + return true + } + ifaces, err := net.InterfaceAddrs() + if err != nil { + return false + } + for _, ia := range ifaces { + var got net.IP + switch v := ia.(type) { + case *net.IPNet: + got = v.IP + case *net.IPAddr: + got = v.IP + } + if got == nil { + continue + } + if strings.EqualFold(got.String(), host) { + return true + } + if ip != nil && got.Equal(ip) { + return true + } + } + return false +} diff --git a/weed/util/net_host_test.go b/weed/util/net_host_test.go new file mode 100644 index 0000000000..7caf30f056 --- /dev/null +++ b/weed/util/net_host_test.go @@ -0,0 +1,24 @@ +package util + +import "testing" + +func TestHostKeyStripsPort(t *testing.T) { + t.Parallel() + if got := HostKey("10.87.0.3:8080"); got != "10.87.0.3" { + t.Fatalf("HostKey replica: got %q", got) + } + if got := HostKey("10.87.0.3:33646"); got != "10.87.0.3" { + t.Fatalf("HostKey worker: got %q", got) + } + if HostKey("10.87.0.3:8080") != HostKey("10.87.0.3:33646") { + t.Fatal("worker and volume server on the same IP must share a HostKey") + } +} + +func TestHostKeysFromAddressesDedups(t *testing.T) { + t.Parallel() + got := HostKeysFromAddresses([]string{"10.0.0.1:8080", "10.0.0.1:8081", "10.0.0.2:8080", ""}) + if len(got) != 2 || got[0] != "10.0.0.1" || got[1] != "10.0.0.2" { + t.Fatalf("got %v", got) + } +} diff --git a/weed/worker/tasks/erasure_coding/ec_task.go b/weed/worker/tasks/erasure_coding/ec_task.go index b4ba0f126b..e86cbf05c6 100644 --- a/weed/worker/tasks/erasure_coding/ec_task.go +++ b/weed/worker/tasks/erasure_coding/ec_task.go @@ -21,6 +21,7 @@ import ( storagetypes "github.com/seaweedfs/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/storage/volume_info" "github.com/seaweedfs/seaweedfs/weed/storage/volume_replica" + "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/wdclient" "github.com/seaweedfs/seaweedfs/weed/worker/types" "github.com/seaweedfs/seaweedfs/weed/worker/types/base" @@ -36,6 +37,9 @@ type ErasureCodingTask struct { workDir string progress float64 grpcDialOption grpc.DialOption + // workerAddress is weed worker -address (host:port). Not a volume URL; + // late-bind matches replica hostnames against HostKey(workerAddress). + workerAddress string // EC parameters dataShards int32 @@ -403,9 +407,75 @@ func (t *ErasureCodingTask) syncAndSelectSourceReplica() error { glog.Infof("EC encode: using best replica %s as source for volume %d (was %s)", best.Url, t.volumeID, t.server) t.server = best.Url } + // Late-bind from after consistency: worker -address hostname, not Sources[0] + // and not AddressIsLocal (replica URLs are hostnames like full-02:8081). + workerAddr := strings.TrimSpace(t.workerAddress) + if workerAddr == "" { + workerAddr, _ = os.Hostname() + } + if local := preferWorkerHostReplicaIfComplete(t.grpcDialOption, needle.VolumeId(t.volumeID), locs, t.server, workerAddr); local != "" && local != t.server { + glog.Infof("EC encode: preferring worker-host replica %s as source for volume %d (was %s)", local, t.volumeID, t.server) + t.server = local + } return nil } +type replicaFileCount struct { + url string + fileCount uint64 + err error +} + +// preferWorkerHostReplicaIfComplete rebinds from to a same-hostname replica +// only when live FileCounts are equal. Diverged counts keep current (max). +func preferWorkerHostReplicaIfComplete(dial grpc.DialOption, vid needle.VolumeId, locs []wdclient.Location, current, workerAddress string) string { + statuses := volume_replica.GetReplicaStatuses(dial, vid, locs) + replicas := make([]replicaFileCount, 0, len(statuses)) + for _, s := range statuses { + replicas = append(replicas, replicaFileCount{url: s.Location.Url, fileCount: s.FileCount, err: s.Error}) + } + return lateBindSourceToWorkerHost(current, workerAddress, replicas) +} + +// lateBindSourceToWorkerHost: equal FileCount → replica on worker -address host +// (any port). Diverge → keep current (already max FileCount). Worker address +// is not a volume URL. +func lateBindSourceToWorkerHost(current, workerAddress string, replicas []replicaFileCount) string { + workerHost := util.HostKey(workerAddress) + if workerHost == "" { + return current + } + var live []replicaFileCount + for _, r := range replicas { + if r.err != nil || r.url == "" { + continue + } + live = append(live, r) + } + if len(live) == 0 { + return current + } + maxCount := live[0].fileCount + allSame := true + for _, r := range live[1:] { + if r.fileCount != live[0].fileCount { + allSame = false + } + if r.fileCount > maxCount { + maxCount = r.fileCount + } + } + if !allSame { + return current + } + for _, r := range live { + if util.HostKey(r.url) == workerHost { + return r.url + } + } + return current +} + // rollbackReadonly is a best-effort restore of every replica markReplicasReadonly // touched, used when the EC task fails before the originals are deleted. Logs but // does not return errors; uses a fresh context since the caller's may be cancelled. diff --git a/weed/worker/tasks/erasure_coding/ec_task_source_host_test.go b/weed/worker/tasks/erasure_coding/ec_task_source_host_test.go new file mode 100644 index 0000000000..e2cc65b1bf --- /dev/null +++ b/weed/worker/tasks/erasure_coding/ec_task_source_host_test.go @@ -0,0 +1,31 @@ +package erasure_coding + +import "testing" + +func TestLateBindSourceToWorkerHostEqualFileCountPrefersWorkerHost(t *testing.T) { + a := replicaFileCount{url: "full-01:8080", fileCount: 21} + b := replicaFileCount{url: "full-02:8081", fileCount: 21} + worker := "full-02:9334" + + // Sources [A,B] — topology-first is A; worker on B must still copy B. + if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{a, b}); got != b.url { + t.Fatalf("Sources [A,B] worker B: got %s want %s", got, b.url) + } + // Invert Sources — still B. + if got := lateBindSourceToWorkerHost(b.url, worker, []replicaFileCount{b, a}); got != b.url { + t.Fatalf("Sources [B,A] worker B: got %s want %s", got, b.url) + } +} + +func TestLateBindSourceToWorkerHostDivergeKeepsMaxFileCount(t *testing.T) { + a := replicaFileCount{url: "full-01:8080", fileCount: 40} + b := replicaFileCount{url: "full-02:8081", fileCount: 21} + worker := "full-02:9334" + + if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{a, b}); got != a.url { + t.Fatalf("diverge [A,B]: got %s want max FileCount %s", got, a.url) + } + if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{b, a}); got != a.url { + t.Fatalf("diverge [B,A]: got %s want max FileCount %s", got, a.url) + } +} diff --git a/weed/worker/tasks/erasure_coding/plugin_handler.go b/weed/worker/tasks/erasure_coding/plugin_handler.go index 1b8f74585e..36ffd5a59e 100644 --- a/weed/worker/tasks/erasure_coding/plugin_handler.go +++ b/weed/worker/tasks/erasure_coding/plugin_handler.go @@ -26,7 +26,9 @@ func init() { Category: pluginworker.CategoryHeavy, Aliases: []string{"erasure-coding", "erasure.coding", "ec"}, Build: func(opts pluginworker.HandlerBuildOptions) (pluginworker.JobHandler, error) { - return NewErasureCodingHandler(opts.GrpcDialOption, opts.WorkingDir), nil + h := NewErasureCodingHandler(opts.GrpcDialOption, opts.WorkingDir) + h.workerAddress = strings.TrimSpace(opts.WorkerAddress) + return h, nil }, }) } @@ -39,6 +41,7 @@ type erasureCodingWorkerConfig struct { type ErasureCodingHandler struct { grpcDialOption grpc.DialOption workingDir string + workerAddress string } func NewErasureCodingHandler(grpcDialOption grpc.DialOption, workingDir string) *ErasureCodingHandler { @@ -519,6 +522,7 @@ func (h *ErasureCodingHandler) Execute( params.Collection, h.grpcDialOption, ) + task.workerAddress = h.workerAddress execCtx, execCancel := context.WithCancel(ctx) defer execCancel() task.SetProgressCallback(func(progress float64, stage string) { @@ -670,8 +674,15 @@ func buildErasureCodingProposal( } sourceNode := "" - if len(params.Sources) > 0 { - sourceNode = strings.TrimSpace(params.Sources[0].Node) + var replicaServers []string + for _, s := range params.Sources { + if s == nil || strings.TrimSpace(s.Node) == "" || len(s.ShardIds) > 0 { + continue + } + replicaServers = append(replicaServers, s.Node) + if sourceNode == "" { + sourceNode = strings.TrimSpace(s.Node) + } } summary := fmt.Sprintf("Erasure code volume %d", result.VolumeID) @@ -702,6 +713,9 @@ func buildErasureCodingProposal( "source_server": { Kind: &plugin_pb.ConfigValue_StringValue{StringValue: sourceNode}, }, + "replica_servers": { + Kind: &plugin_pb.ConfigValue_StringList{StringList: &plugin_pb.StringList{Values: replicaServers}}, + }, "collection": { Kind: &plugin_pb.ConfigValue_StringValue{StringValue: result.Collection}, },