mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 00:38:53 +01:00
* s3 lifecycle: add a LifecycleDelete client that fails over between S3 fronts When the current front returns Unavailable, the client closes it, marks it failed for the rest of the pass, picks the next front that passes checkLifecycleReady and resends the same request there. Callers cut off by that switch closing their connection resend on the new front. When no front is left, every later call returns one error naming the fronts tried. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: serve the pass through the failover client Execute hands the replay drain and the walker one failover client built on the selected front, reports each switch as job progress, and closes the current connection when the pass ends. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: keep failing over until a front serves the resent delete The resend after a switch went straight to the new front, so a second lost front ended the call even with a healthy third one, and the walker has no retry of its own. Resend through the failover loop instead. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3 lifecycle: do not recheck fronts skipped when the pass started The failover client started with no failed fronts, so losing the selected front checked the fronts selectS3Front had already skipped again, holding every delete for another readiness timeout per front. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
133 lines
3.5 KiB
Go
133 lines
3.5 KiB
Go
package s3_lifecycle
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
)
|
|
|
|
// failoverClient sends LifecycleDelete to one S3 front and, when that front
|
|
// becomes unreachable, moves the pass to the next front that passes
|
|
// checkLifecycleReady. A front that failed, including one selectS3Front
|
|
// skipped before addr, is not selected again in the pass.
|
|
type failoverClient struct {
|
|
ctx context.Context
|
|
h *Handler
|
|
endpoints []string
|
|
report func(string)
|
|
|
|
mu sync.Mutex
|
|
conn *grpc.ClientConn
|
|
addr string
|
|
client lifecycleRPCAdapter
|
|
failed map[string]bool
|
|
exhausted error
|
|
}
|
|
|
|
func (h *Handler) newFailoverClient(ctx context.Context, conn *grpc.ClientConn, addr string, endpoints []string, report func(string)) *failoverClient {
|
|
f := &failoverClient{ctx: ctx, h: h, endpoints: endpoints, report: report, failed: map[string]bool{}}
|
|
for _, e := range endpoints {
|
|
if e == addr {
|
|
break
|
|
}
|
|
f.failed[e] = true
|
|
}
|
|
f.use(conn, addr)
|
|
return f
|
|
}
|
|
|
|
func (f *failoverClient) use(conn *grpc.ClientConn, addr string) {
|
|
f.conn, f.addr = conn, addr
|
|
f.client = lifecycleRPCAdapter{c: s3_lifecycle_pb.NewSeaweedS3LifecycleInternalClient(conn)}
|
|
}
|
|
|
|
// LifecycleDelete resends the same request after each switch: the server's
|
|
// identity check resolves a delete that landed before the reply was lost
|
|
// instead of applying it twice. Every switch retires a front, which bounds
|
|
// the loop.
|
|
func (f *failoverClient) LifecycleDelete(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
|
|
for {
|
|
f.mu.Lock()
|
|
conn, client, exhausted := f.conn, f.client, f.exhausted
|
|
f.mu.Unlock()
|
|
if exhausted != nil {
|
|
return nil, exhausted
|
|
}
|
|
resp, err := client.LifecycleDelete(ctx, req)
|
|
if code := status.Code(err); err == nil || ctx.Err() != nil || (code != codes.Unavailable && code != codes.Canceled) {
|
|
return resp, err
|
|
}
|
|
if err := f.failover(conn, err); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
|
|
// failover replaces the front behind failing when it is unavailable; a call
|
|
// cut off by an earlier switch closing its connection just moves on.
|
|
func (f *failoverClient) failover(failing *grpc.ClientConn, cause error) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.exhausted != nil {
|
|
return f.exhausted
|
|
}
|
|
if f.conn != failing {
|
|
return nil
|
|
}
|
|
code := status.Code(cause)
|
|
if code != codes.Unavailable {
|
|
return cause
|
|
}
|
|
from := f.addr
|
|
f.failed[from] = true
|
|
f.conn.Close()
|
|
f.conn = nil
|
|
|
|
var remaining []string
|
|
for _, addr := range f.endpoints {
|
|
if !f.failed[addr] {
|
|
remaining = append(remaining, addr)
|
|
}
|
|
}
|
|
conn, addr, skipped := f.h.selectS3Front(f.ctx, remaining)
|
|
if f.ctx.Err() == nil {
|
|
for _, s := range remaining {
|
|
if s == addr {
|
|
break
|
|
}
|
|
f.failed[s] = true
|
|
}
|
|
}
|
|
lost := fmt.Errorf("s3 %s: %s", from, code)
|
|
if conn == nil {
|
|
f.exhausted = noReadyS3Error(append([]error{lost}, skipped...))
|
|
glog.Warningf("s3 lifecycle: %v", f.exhausted)
|
|
f.report(f.exhausted.Error())
|
|
return f.exhausted
|
|
}
|
|
f.use(conn, addr)
|
|
msg := fmt.Sprintf("switched to s3 %s, %v", addr, lost)
|
|
if len(skipped) > 0 {
|
|
msg += fmt.Sprintf(", skipped: %v", errors.Join(skipped...))
|
|
}
|
|
glog.Warningf("s3 lifecycle: %s", msg)
|
|
f.report(msg)
|
|
return nil
|
|
}
|
|
|
|
func (f *failoverClient) Close() {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.conn != nil {
|
|
f.conn.Close()
|
|
f.conn = nil
|
|
}
|
|
}
|