mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 00:38:53 +01:00
s3 lifecycle: fail over between S3 fronts during a pass (#12647)
* s3 lifecycle: add a LifecycleDelete client that fails over between S3 fronts When the current front returns Unavailable, the client closes it, marks it failed for the rest of the pass, picks the next front that passes checkLifecycleReady and resends the same request there. Callers cut off by that switch closing their connection resend on the new front. When no front is left, every later call returns one error naming the fronts tried. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: serve the pass through the failover client Execute hands the replay drain and the walker one failover client built on the selected front, reports each switch as job progress, and closes the current connection when the pass ends. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: keep failing over until a front serves the resent delete The resend after a switch went straight to the new front, so a second lost front ended the call even with a healthy third one, and the walker has no retry of its own. Resend through the failover loop instead. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: do not recheck fronts skipped when the pass started The failover client started with no failed fronts, so losing the selected front checked the fronts selectS3Front had already skipped again, holding every delete for another readiness timeout per front. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a91b333658
commit
799fd10342
3 files changed
+457
-11
No files matched your search
@@ -254,7 +254,14 @@ func (h *Handler) Execute(ctx context.Context, request *plugin_pb.ExecuteJobRequ
|
||||
if s3Conn == nil {
|
||||
return noReadyS3Error(skipped)
|
||||
}
|
||||
defer s3Conn.Close()
|
||||
client := h.newFailoverClient(runCtx, s3Conn, s3Endpoint, s3Endpoints, func(msg string) {
|
||||
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: request.Job.JobId, JobType: jobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_RUNNING, Stage: "running",
|
||||
Message: msg,
|
||||
})
|
||||
})
|
||||
defer client.Close()
|
||||
if len(skipped) > 0 {
|
||||
skippedErr := errors.Join(skipped...)
|
||||
glog.Warningf("s3 lifecycle: using s3 %s, skipped: %v", s3Endpoint, skippedErr)
|
||||
@@ -264,9 +271,8 @@ func (h *Handler) Execute(ctx context.Context, request *plugin_pb.ExecuteJobRequ
|
||||
Message: fmt.Sprintf("using s3 %s, skipped: %v", s3Endpoint, skippedErr),
|
||||
})
|
||||
}
|
||||
rpc := s3_lifecycle_pb.NewSeaweedS3LifecycleInternalClient(s3Conn)
|
||||
|
||||
report, err := h.executeDailyReplay(runCtx, request, bucketsPath, filerClient, rpc, cfg, sender)
|
||||
report, err := h.executeDailyReplay(runCtx, request, bucketsPath, filerClient, client, cfg, sender)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -296,7 +302,7 @@ func sendCompletion(request *plugin_pb.ExecuteJobRequest, sender pluginworker.Ex
|
||||
// and against the steady-state walk view; all rule kinds are serviced.
|
||||
// The returned error covers setup only; the pass's own error is in the
|
||||
// report.
|
||||
func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.ExecuteJobRequest, bucketsPath string, filerClient filer_pb.SeaweedFilerClient, rpc s3_lifecycle_pb.SeaweedS3LifecycleInternalClient, cfg Config, sender pluginworker.ExecutionSender) (*runReport, error) {
|
||||
func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.ExecuteJobRequest, bucketsPath string, filerClient filer_pb.SeaweedFilerClient, client dailyrun.LifecycleClient, cfg Config, sender pluginworker.ExecutionSender) (*runReport, error) {
|
||||
startedAt := time.Now()
|
||||
eng := engine.New()
|
||||
inputs, parseErrors, err := scheduler.LoadCompileInputs(ctx, filerClient, bucketsPath)
|
||||
@@ -315,10 +321,6 @@ func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.Exe
|
||||
|
||||
limiter, limiterDesc := buildLimiterFromClusterContext(request.GetClusterContext())
|
||||
|
||||
// Reuse one LifecycleClient across the replay drain and the
|
||||
// walker's per-entry dispatch.
|
||||
client := lifecycleRPCAdapter{c: rpc}
|
||||
|
||||
// Bucket list for the walker — derived from inputs so it matches
|
||||
// the snapshot the engine compiled.
|
||||
buckets := make([]string, 0, len(inputs))
|
||||
@@ -463,9 +465,9 @@ func parsePositiveInt(s string) (int, bool) {
|
||||
}
|
||||
|
||||
// clusterS3Endpoints returns the master-discovered S3 gRPC addresses for the
|
||||
// cluster. The handler uses the first one that passes checkLifecycleReady;
|
||||
// the master refreshes the list on KeepConnected so a stale entry self-heals
|
||||
// on the next run.
|
||||
// cluster. The handler uses the first one that passes checkLifecycleReady and
|
||||
// moves to the next ready one if it becomes unavailable during the pass; the
|
||||
// master refreshes the list on KeepConnected.
|
||||
func clusterS3Endpoints(cc *plugin_pb.ClusterContext) []string {
|
||||
if cc == nil {
|
||||
return nil
|
||||
|
||||
Reference in new issue
Block a user