s3 lifecycle: fail over between S3 fronts during a pass (#12647)

* s3 lifecycle: add a LifecycleDelete client that fails over between S3 fronts

When the current front returns Unavailable, the client closes it, marks it
failed for the rest of the pass, picks the next front that passes
checkLifecycleReady and resends the same request there. Callers cut off by
that switch closing their connection resend on the new front. When no front
is left, every later call returns one error naming the fronts tried.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3 lifecycle: serve the pass through the failover client

Execute hands the replay drain and the walker one failover client built on
the selected front, reports each switch as job progress, and closes the
current connection when the pass ends.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3 lifecycle: keep failing over until a front serves the resent delete

The resend after a switch went straight to the new front, so a second
lost front ended the call even with a healthy third one, and the walker
has no retry of its own. Resend through the failover loop instead.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3 lifecycle: do not recheck fronts skipped when the pass started

The failover client started with no failed fronts, so losing the selected
front checked the fronts selectS3Front had already skipped again, holding
every delete for another readiness timeout per front.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed 2026-11-15 12:37:07 -08:00
1 parent a91b333658
commit 799fd10342
3 files changed
+457 -11

No files matched your search

+132
View File
@@ -0,0 +1,132 @@
package s3_lifecycle
import (
"context"
"errors"
"fmt"
"sync"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// failoverClient sends LifecycleDelete to one S3 front and, when that front
// becomes unreachable, moves the pass to the next front that passes
// checkLifecycleReady. A front that failed, including one selectS3Front
// skipped before addr, is not selected again in the pass.
type failoverClient struct {
ctx context.Context
h *Handler
endpoints []string
report func(string)
mu sync.Mutex
conn *grpc.ClientConn
addr string
client lifecycleRPCAdapter
failed map[string]bool
exhausted error
}
func (h *Handler) newFailoverClient(ctx context.Context, conn *grpc.ClientConn, addr string, endpoints []string, report func(string)) *failoverClient {
f := &failoverClient{ctx: ctx, h: h, endpoints: endpoints, report: report, failed: map[string]bool{}}
for _, e := range endpoints {
if e == addr {
break
}
f.failed[e] = true
}
f.use(conn, addr)
return f
}
func (f *failoverClient) use(conn *grpc.ClientConn, addr string) {
f.conn, f.addr = conn, addr
f.client = lifecycleRPCAdapter{c: s3_lifecycle_pb.NewSeaweedS3LifecycleInternalClient(conn)}
}
// LifecycleDelete resends the same request after each switch: the server's
// identity check resolves a delete that landed before the reply was lost
// instead of applying it twice. Every switch retires a front, which bounds
// the loop.
func (f *failoverClient) LifecycleDelete(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
for {
f.mu.Lock()
conn, client, exhausted := f.conn, f.client, f.exhausted
f.mu.Unlock()
if exhausted != nil {
return nil, exhausted
}
resp, err := client.LifecycleDelete(ctx, req)
if code := status.Code(err); err == nil || ctx.Err() != nil || (code != codes.Unavailable && code != codes.Canceled) {
return resp, err
}
if err := f.failover(conn, err); err != nil {
return nil, err
}
}
}
// failover replaces the front behind failing when it is unavailable; a call
// cut off by an earlier switch closing its connection just moves on.
func (f *failoverClient) failover(failing *grpc.ClientConn, cause error) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.exhausted != nil {
return f.exhausted
}
if f.conn != failing {
return nil
}
code := status.Code(cause)
if code != codes.Unavailable {
return cause
}
from := f.addr
f.failed[from] = true
f.conn.Close()
f.conn = nil
var remaining []string
for _, addr := range f.endpoints {
if !f.failed[addr] {
remaining = append(remaining, addr)
}
}
conn, addr, skipped := f.h.selectS3Front(f.ctx, remaining)
if f.ctx.Err() == nil {
for _, s := range remaining {
if s == addr {
break
}
f.failed[s] = true
}
}
lost := fmt.Errorf("s3 %s: %s", from, code)
if conn == nil {
f.exhausted = noReadyS3Error(append([]error{lost}, skipped...))
glog.Warningf("s3 lifecycle: %v", f.exhausted)
f.report(f.exhausted.Error())
return f.exhausted
}
f.use(conn, addr)
msg := fmt.Sprintf("switched to s3 %s, %v", addr, lost)
if len(skipped) > 0 {
msg += fmt.Sprintf(", skipped: %v", errors.Join(skipped...))
}
glog.Warningf("s3 lifecycle: %s", msg)
f.report(msg)
return nil
}
func (f *failoverClient) Close() {
f.mu.Lock()
defer f.mu.Unlock()
if f.conn != nil {
f.conn.Close()
f.conn = nil
}
}
@@ -0,0 +1,312 @@
package s3_lifecycle
import (
"context"
"fmt"
"net"
"strings"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
"google.golang.org/grpc"
"google.golang.org/protobuf/proto"
)
type failoverFront struct {
testFront
dieOn int
hold bool
stop func()
mu sync.Mutex
reqs []*s3_lifecycle_pb.LifecycleDeleteRequest
}
// LifecycleDelete applies the request; on the dieOn-th call the server then
// stops before replying, like a front lost after the delete landed.
func (f *failoverFront) LifecycleDelete(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
if f.hold {
<-ctx.Done()
return nil, ctx.Err()
}
resp, err := f.testFront.LifecycleDelete(ctx, req)
f.mu.Lock()
f.reqs = append(f.reqs, proto.Clone(req).(*s3_lifecycle_pb.LifecycleDeleteRequest))
n := len(f.reqs)
f.mu.Unlock()
if n == f.dieOn {
go f.stop()
<-ctx.Done()
return nil, ctx.Err()
}
return resp, err
}
func (f *failoverFront) received() []*s3_lifecycle_pb.LifecycleDeleteRequest {
f.mu.Lock()
defer f.mu.Unlock()
return append([]*s3_lifecycle_pb.LifecycleDeleteRequest(nil), f.reqs...)
}
func startFailoverFront(t *testing.T, f *failoverFront) string {
t.Helper()
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
s := grpc.NewServer()
s3_lifecycle_pb.RegisterSeaweedS3LifecycleInternalServer(s, f)
f.stop = s.Stop
go s.Serve(lis)
t.Cleanup(s.Stop)
return lis.Addr().String()
}
func downAddr(t *testing.T) string {
t.Helper()
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
addr := lis.Addr().String()
lis.Close()
return addr
}
type reports struct {
mu sync.Mutex
msgs []string
}
func (r *reports) add(msg string) {
r.mu.Lock()
defer r.mu.Unlock()
r.msgs = append(r.msgs, msg)
}
func (r *reports) all() []string {
r.mu.Lock()
defer r.mu.Unlock()
return append([]string(nil), r.msgs...)
}
func newTestFailoverClient(t *testing.T, ctx context.Context, endpoints []string, r *reports) *failoverClient {
t.Helper()
h := newTestHandler()
conn, addr, skipped := h.selectS3Front(ctx, endpoints)
if conn == nil {
t.Fatalf("no ready front: %v", skipped)
}
f := h.newFailoverClient(ctx, conn, addr, endpoints, r.add)
t.Cleanup(f.Close)
return f
}
func deleteReq(i int) *s3_lifecycle_pb.LifecycleDeleteRequest {
return &s3_lifecycle_pb.LifecycleDeleteRequest{
Bucket: "bk",
ObjectPath: fmt.Sprintf("secret-key-%d", i),
VersionId: fmt.Sprintf("v%d", i),
RuleHash: []byte{1, 2, 3},
ActionKind: s3_lifecycle_pb.ActionKind_NONCURRENT_DAYS,
ExpectedIdentity: &s3_lifecycle_pb.EntryIdentity{MtimeNs: int64(i), Size: 7, HeadFid: "3,01"},
}
}
func TestSelectS3Front_FirstFrontDownUsesNext(t *testing.T) {
setWorkerKey(t, frontKey)
down := downAddr(t)
goodAddr := startFront(t, &testFront{key: frontKey})
conn, got, skipped := newTestHandler().selectS3Front(context.Background(), []string{down, goodAddr})
if conn == nil {
t.Fatalf("expected the second front, skipped: %v", skipped)
}
conn.Close()
if got != goodAddr || len(skipped) != 1 || !strings.Contains(skipped[0].Error(), down) {
t.Fatalf("got %q skipped %v, want %q after skipping %s", got, skipped, goodAddr, down)
}
}
func TestFailoverClient_FrontLostMidPassResendsSameRequest(t *testing.T) {
setWorkerKey(t, frontKey)
a := &failoverFront{testFront: testFront{key: frontKey}, dieOn: 3}
b := &failoverFront{testFront: testFront{key: frontKey}}
aAddr, bAddr := startFailoverFront(t, a), startFailoverFront(t, b)
r := &reports{}
f := newTestFailoverClient(t, context.Background(), []string{aAddr, bAddr}, r)
for i := 1; i <= 5; i++ {
resp, err := f.LifecycleDelete(context.Background(), deleteReq(i))
if err != nil || resp.Outcome != s3_lifecycle_pb.LifecycleDeleteOutcome_DONE {
t.Fatalf("delete %d: resp %v err %v", i, resp, err)
}
}
gotA, gotB := a.received(), b.received()
if len(gotA) != 3 || len(gotB) != 3 {
t.Fatalf("front a got %d, front b got %d deletes; want 3 and 3", len(gotA), len(gotB))
}
if !proto.Equal(gotA[2], gotB[0]) || !proto.Equal(gotB[0], deleteReq(3)) {
t.Fatalf("resent request differs: a=%v b=%v", gotA[2], gotB[0])
}
msgs := r.all()
if len(msgs) != 1 || !strings.Contains(msgs[0], "switched to s3 "+bAddr) || !strings.Contains(msgs[0], aAddr) || !strings.Contains(msgs[0], "Unavailable") {
t.Fatalf("want one switch report naming both fronts, got %q", msgs)
}
for _, m := range msgs {
assertNoSecrets(t, m)
if strings.Contains(m, "secret-key") {
t.Fatalf("report exposes an object key: %s", m)
}
}
}
func TestFailoverClient_ConcurrentCallersSwitchOnce(t *testing.T) {
setWorkerKey(t, frontKey)
a := &failoverFront{testFront: testFront{key: frontKey}, dieOn: 4}
b := &failoverFront{testFront: testFront{key: frontKey}}
c := &failoverFront{testFront: testFront{key: frontKey}}
aAddr, bAddr, cAddr := startFailoverFront(t, a), startFailoverFront(t, b), startFailoverFront(t, c)
r := &reports{}
f := newTestFailoverClient(t, context.Background(), []string{aAddr, bAddr, cAddr}, r)
var wg sync.WaitGroup
errs := make(chan error, 40)
for i := 0; i < 40; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
if _, err := f.LifecycleDelete(context.Background(), deleteReq(i)); err != nil {
errs <- err
}
}(i)
}
wg.Wait()
close(errs)
for err := range errs {
t.Errorf("delete failed: %v", err)
}
if msgs := r.all(); len(msgs) != 1 || !strings.Contains(msgs[0], "switched to s3 "+bAddr) {
t.Fatalf("want one switch to %s, got %q", bAddr, msgs)
}
if n := len(c.received()); n != 0 {
t.Fatalf("front c got %d deletes, want none", n)
}
}
func TestFailoverClient_NoFrontLeftFailsFast(t *testing.T) {
setWorkerKey(t, frontKey)
a := &failoverFront{testFront: testFront{key: frontKey}, dieOn: 1}
aAddr := startFailoverFront(t, a)
down := downAddr(t)
rejecting := &testFront{key: otherKey}
rejectAddr := startFront(t, rejecting)
r := &reports{}
f := newTestFailoverClient(t, context.Background(), []string{aAddr, down, rejectAddr}, r)
start := time.Now()
_, err := f.LifecycleDelete(context.Background(), deleteReq(1))
if err == nil {
t.Fatal("want an error with no front left")
}
msg := err.Error()
for _, want := range []string{"no s3 server is ready for lifecycle deletes", aAddr, down, rejectAddr, "rejected lifecycle credentials", "jwt.filer_signing.key"} {
if !strings.Contains(msg, want) {
t.Fatalf("error %q does not mention %q", msg, want)
}
}
assertNoSecrets(t, msg)
if strings.Contains(msg, "secret-key") {
t.Fatalf("error exposes an object key: %s", msg)
}
if _, err2 := f.LifecycleDelete(context.Background(), deleteReq(2)); err2 == nil || err2.Error() != msg {
t.Fatalf("second call: want the same error, got %v", err2)
}
if elapsed := time.Since(start); elapsed > 10*time.Second {
t.Fatalf("giving up took %v", elapsed)
}
if n := rejecting.readies.Load(); n != 1 {
t.Fatalf("rejecting front checked %d times, want once", n)
}
if msgs := r.all(); len(msgs) != 1 || msgs[0] != msg {
t.Fatalf("want the error reported once, got %q", msgs)
}
}
func TestFailoverClient_ParentCancelDoesNotSwitch(t *testing.T) {
setWorkerKey(t, frontKey)
a := &failoverFront{testFront: testFront{key: frontKey}, hold: true}
b := &failoverFront{testFront: testFront{key: frontKey}}
aAddr, bAddr := startFailoverFront(t, a), startFailoverFront(t, b)
r := &reports{}
f := newTestFailoverClient(t, context.Background(), []string{aAddr, bAddr}, r)
ctx, cancel := context.WithCancel(context.Background())
time.AfterFunc(100*time.Millisecond, cancel)
start := time.Now()
if _, err := f.LifecycleDelete(ctx, deleteReq(1)); err == nil {
t.Fatal("want an error after cancellation")
}
if elapsed := time.Since(start); elapsed > 5*time.Second {
t.Fatalf("cancelled delete took %v", elapsed)
}
if b.readies.Load() != 0 || len(r.all()) != 0 {
t.Fatalf("switched fronts after cancellation: %d readiness checks, reports %q", b.readies.Load(), r.all())
}
f.mu.Lock()
current := f.addr
f.mu.Unlock()
if current != aAddr {
t.Fatalf("current front %q, want %q", current, aAddr)
}
}
func TestFailoverClient_SecondFrontLostMovesToThird(t *testing.T) {
setWorkerKey(t, frontKey)
a := &failoverFront{testFront: testFront{key: frontKey}, dieOn: 1}
b := &failoverFront{testFront: testFront{key: frontKey}, dieOn: 1}
c := &failoverFront{testFront: testFront{key: frontKey}}
aAddr, bAddr, cAddr := startFailoverFront(t, a), startFailoverFront(t, b), startFailoverFront(t, c)
r := &reports{}
f := newTestFailoverClient(t, context.Background(), []string{aAddr, bAddr, cAddr}, r)
resp, err := f.LifecycleDelete(context.Background(), deleteReq(1))
if err != nil || resp.Outcome != s3_lifecycle_pb.LifecycleDeleteOutcome_DONE {
t.Fatalf("resp %v err %v", resp, err)
}
for name, front := range map[string]*failoverFront{"a": a, "b": b, "c": c} {
got := front.received()
if len(got) != 1 || !proto.Equal(got[0], deleteReq(1)) {
t.Fatalf("front %s got %v, want the one request", name, got)
}
}
if msgs := r.all(); len(msgs) != 2 || !strings.Contains(msgs[1], "switched to s3 "+cAddr) {
t.Fatalf("want switches to b then c, got %q", msgs)
}
}
func TestFailoverClient_SkipsFrontSkippedAtStartup(t *testing.T) {
setWorkerKey(t, frontKey)
prev := lifecycleReadyTimeout
lifecycleReadyTimeout = 200 * time.Millisecond
t.Cleanup(func() { lifecycleReadyTimeout = prev })
stuck := &failoverFront{testFront: testFront{key: frontKey, block: true}}
b := &failoverFront{testFront: testFront{key: frontKey}, dieOn: 1}
c := &failoverFront{testFront: testFront{key: frontKey}}
stuckAddr, bAddr, cAddr := startFailoverFront(t, stuck), startFailoverFront(t, b), startFailoverFront(t, c)
r := &reports{}
f := newTestFailoverClient(t, context.Background(), []string{stuckAddr, bAddr, cAddr}, r)
if _, err := f.LifecycleDelete(context.Background(), deleteReq(1)); err != nil {
t.Fatal(err)
}
if n := stuck.readies.Load(); n != 1 {
t.Fatalf("front skipped at startup checked %d times, want once", n)
}
if msgs := r.all(); len(msgs) != 1 || !strings.Contains(msgs[0], "switched to s3 "+cAddr) || strings.Contains(msgs[0], "skipped") {
t.Fatalf("want a direct switch to %s, got %q", cAddr, msgs)
}
}
+13 -11
View File
@@ -254,7 +254,14 @@ func (h *Handler) Execute(ctx context.Context, request *plugin_pb.ExecuteJobRequ
if s3Conn == nil {
return noReadyS3Error(skipped)
}
defer s3Conn.Close()
client := h.newFailoverClient(runCtx, s3Conn, s3Endpoint, s3Endpoints, func(msg string) {
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
JobId: request.Job.JobId, JobType: jobType,
State: plugin_pb.JobState_JOB_STATE_RUNNING, Stage: "running",
Message: msg,
})
})
defer client.Close()
if len(skipped) > 0 {
skippedErr := errors.Join(skipped...)
glog.Warningf("s3 lifecycle: using s3 %s, skipped: %v", s3Endpoint, skippedErr)
@@ -264,9 +271,8 @@ func (h *Handler) Execute(ctx context.Context, request *plugin_pb.ExecuteJobRequ
Message: fmt.Sprintf("using s3 %s, skipped: %v", s3Endpoint, skippedErr),
})
}
rpc := s3_lifecycle_pb.NewSeaweedS3LifecycleInternalClient(s3Conn)
report, err := h.executeDailyReplay(runCtx, request, bucketsPath, filerClient, rpc, cfg, sender)
report, err := h.executeDailyReplay(runCtx, request, bucketsPath, filerClient, client, cfg, sender)
if err != nil {
return err
}
@@ -296,7 +302,7 @@ func sendCompletion(request *plugin_pb.ExecuteJobRequest, sender pluginworker.Ex
// and against the steady-state walk view; all rule kinds are serviced.
// The returned error covers setup only; the pass's own error is in the
// report.
func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.ExecuteJobRequest, bucketsPath string, filerClient filer_pb.SeaweedFilerClient, rpc s3_lifecycle_pb.SeaweedS3LifecycleInternalClient, cfg Config, sender pluginworker.ExecutionSender) (*runReport, error) {
func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.ExecuteJobRequest, bucketsPath string, filerClient filer_pb.SeaweedFilerClient, client dailyrun.LifecycleClient, cfg Config, sender pluginworker.ExecutionSender) (*runReport, error) {
startedAt := time.Now()
eng := engine.New()
inputs, parseErrors, err := scheduler.LoadCompileInputs(ctx, filerClient, bucketsPath)
@@ -315,10 +321,6 @@ func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.Exe
limiter, limiterDesc := buildLimiterFromClusterContext(request.GetClusterContext())
// Reuse one LifecycleClient across the replay drain and the
// walker's per-entry dispatch.
client := lifecycleRPCAdapter{c: rpc}
// Bucket list for the walker — derived from inputs so it matches
// the snapshot the engine compiled.
buckets := make([]string, 0, len(inputs))
@@ -463,9 +465,9 @@ func parsePositiveInt(s string) (int, bool) {
}
// clusterS3Endpoints returns the master-discovered S3 gRPC addresses for the
// cluster. The handler uses the first one that passes checkLifecycleReady;
// the master refreshes the list on KeepConnected so a stale entry self-heals
// on the next run.
// cluster. The handler uses the first one that passes checkLifecycleReady and
// moves to the next ready one if it becomes unavailable during the pass; the
// master refreshes the list on KeepConnected.
func clusterS3Endpoints(cc *plugin_pb.ClusterContext) []string {
if cc == nil {
return nil