Files
seaweedfs/weed/worker/tasks/s3_lifecycle/readiness_test.go
T
Chris LuandClaude Opus 5.5 858f8cbad8 s3 lifecycle: check internal S3 authentication before starting a pass (#12645)
* s3 lifecycle: add a LifecycleReady RPC that only authorizes the caller

LifecycleReady runs the same checkAdminAuth as LifecycleDelete and reads
no bucket or object metadata, so a lifecycle worker can confirm that an S3
front accepts its internal credentials before it starts a pass.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* s3 lifecycle: check S3 fronts with LifecycleReady before the pass

Execute used to dial the first registered S3 front and start loading
policies and walking buckets, so a missing or mismatched
jwt.filer_signing.key surfaced only as repeated per-bucket delete
failures. It now checks each front in order with LifecycleReady, using
the credentials LifecycleDelete sends, under a 10s bound that also
follows parent cancellation. The first front that passes serves the
pass; fronts that fail are reported in the job's progress and the log.
When none passes, the job fails before any listing with an error that
names the front and says whether this worker has the key set, without
key or token material. Fronts that predate LifecycleReady are checked
with an empty LifecycleDelete, which they authorize and then reject
before any lookup.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 10:53:07 -08:00

281 lines
9.2 KiB
Go

package s3_lifecycle
import (
"context"
"errors"
"net"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/status"
)
const (
frontKey = "readiness-test-front-key"
otherKey = "readiness-test-other-key"
)
type testFront struct {
s3_lifecycle_pb.UnimplementedSeaweedS3LifecycleInternalServer
key string
legacy bool
block bool
readies atomic.Int32
deletes atomic.Int32
}
func (f *testFront) LifecycleReady(ctx context.Context, _ *s3_lifecycle_pb.LifecycleReadyRequest) (*s3_lifecycle_pb.LifecycleReadyResponse, error) {
f.readies.Add(1)
if f.legacy {
return nil, status.Error(codes.Unimplemented, "method LifecycleReady not implemented")
}
if f.block {
<-ctx.Done()
return nil, status.FromContextError(ctx.Err()).Err()
}
if err := security.CheckGrpcAdminToken(ctx, security.SigningKey(f.key)); err != nil {
return nil, err
}
return &s3_lifecycle_pb.LifecycleReadyResponse{}, nil
}
func (f *testFront) LifecycleDelete(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
f.deletes.Add(1)
if err := security.CheckGrpcAdminToken(ctx, security.SigningKey(f.key)); err != nil {
return nil, err
}
if req.Bucket == "" || req.ObjectPath == "" {
return &s3_lifecycle_pb.LifecycleDeleteResponse{Outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_BLOCKED}, nil
}
return &s3_lifecycle_pb.LifecycleDeleteResponse{Outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}, nil
}
type countingFiler struct {
filer_pb.UnimplementedSeaweedFilerServer
lists atomic.Int32
}
func (f *countingFiler) GetFilerConfiguration(context.Context, *filer_pb.GetFilerConfigurationRequest) (*filer_pb.GetFilerConfigurationResponse, error) {
return &filer_pb.GetFilerConfigurationResponse{DirBuckets: "/buckets"}, nil
}
func (f *countingFiler) ListEntries(*filer_pb.ListEntriesRequest, grpc.ServerStreamingServer[filer_pb.ListEntriesResponse]) error {
f.lists.Add(1)
return nil
}
func serveGrpc(t *testing.T, register func(*grpc.Server)) string {
t.Helper()
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
s := grpc.NewServer()
register(s)
go s.Serve(lis)
t.Cleanup(s.Stop)
return lis.Addr().String()
}
func startFront(t *testing.T, f *testFront) string {
return serveGrpc(t, func(s *grpc.Server) { s3_lifecycle_pb.RegisterSeaweedS3LifecycleInternalServer(s, f) })
}
func setWorkerKey(t *testing.T, key string) {
t.Helper()
v := util.GetViper()
prev := v.GetString("jwt.filer_signing.key")
v.Set("jwt.filer_signing.key", key)
t.Cleanup(func() { v.Set("jwt.filer_signing.key", prev) })
}
func newTestHandler() *Handler {
return NewHandler(grpc.WithTransportCredentials(insecure.NewCredentials()))
}
func assertNoSecrets(t *testing.T, msg string) {
t.Helper()
for _, secret := range []string{frontKey, otherKey, "Bearer", "eyJ"} {
if strings.Contains(msg, secret) {
t.Fatalf("error exposes %q: %s", secret, msg)
}
}
}
func TestSelectS3Front_SharedKeyPassesWithoutDeletes(t *testing.T) {
setWorkerKey(t, frontKey)
front := &testFront{key: frontKey}
addr := startFront(t, front)
conn, got, skipped := newTestHandler().selectS3Front(context.Background(), []string{addr})
if conn == nil {
t.Fatalf("expected a ready front, skipped: %v", skipped)
}
conn.Close()
if got != addr || len(skipped) != 0 {
t.Fatalf("got front %q skipped %v, want %q and none", got, skipped, addr)
}
if front.deletes.Load() != 0 {
t.Fatalf("readiness check sent %d LifecycleDelete calls", front.deletes.Load())
}
}
func TestSelectS3Front_KeyProblemsNameTheConfig(t *testing.T) {
cases := []struct {
name, workerKey, wantState string
}{
{"missing worker key", "", "is not set"},
{"mismatched worker key", otherKey, "is set"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
setWorkerKey(t, tc.workerKey)
addr := startFront(t, &testFront{key: frontKey})
conn, _, skipped := newTestHandler().selectS3Front(context.Background(), []string{addr})
if conn != nil {
conn.Close()
t.Fatal("front with a different key was selected")
}
if len(skipped) != 1 || status.Code(errors.Unwrap(skipped[0])) != codes.Unauthenticated {
t.Fatalf("want one Unauthenticated skip, got %v", skipped)
}
msg := noReadyS3Error(skipped).Error()
for _, want := range []string{addr, "jwt.filer_signing.key " + tc.wantState, "security.toml"} {
if !strings.Contains(msg, want) {
t.Fatalf("error %q does not mention %q", msg, want)
}
}
assertNoSecrets(t, msg)
})
}
}
func TestSelectS3Front_SkipsFrontWithOtherKey(t *testing.T) {
setWorkerKey(t, frontKey)
bad := &testFront{key: otherKey}
good := &testFront{key: frontKey}
badAddr, goodAddr := startFront(t, bad), startFront(t, good)
conn, got, skipped := newTestHandler().selectS3Front(context.Background(), []string{badAddr, goodAddr})
if conn == nil {
t.Fatalf("expected the second front, skipped: %v", skipped)
}
conn.Close()
if got != goodAddr {
t.Fatalf("selected %q, want %q", got, goodAddr)
}
if len(skipped) != 1 || !strings.Contains(skipped[0].Error(), badAddr) || !strings.Contains(skipped[0].Error(), "rejected lifecycle credentials") {
t.Fatalf("want the first front reported as a credential problem, got %v", skipped)
}
if bad.deletes.Load() != 0 || good.deletes.Load() != 0 {
t.Fatal("readiness check sent LifecycleDelete calls")
}
}
func TestSelectS3Front_OlderFrontUsesEmptyDelete(t *testing.T) {
setWorkerKey(t, frontKey)
ok := &testFront{key: frontKey, legacy: true}
okAddr := startFront(t, ok)
conn, _, skipped := newTestHandler().selectS3Front(context.Background(), []string{okAddr})
if conn == nil {
t.Fatalf("older front with the shared key was skipped: %v", skipped)
}
conn.Close()
if ok.deletes.Load() != 1 {
t.Fatalf("want one empty LifecycleDelete, got %d", ok.deletes.Load())
}
mismatched := &testFront{key: otherKey, legacy: true}
conn, _, skipped = newTestHandler().selectS3Front(context.Background(), []string{startFront(t, mismatched)})
if conn != nil {
conn.Close()
t.Fatal("older front with another key was selected")
}
if len(skipped) != 1 || !strings.Contains(skipped[0].Error(), "rejected lifecycle credentials") {
t.Fatalf("want a credential problem, got %v", skipped)
}
}
func TestSelectS3Front_TimeoutMovesOn(t *testing.T) {
setWorkerKey(t, frontKey)
prev := lifecycleReadyTimeout
lifecycleReadyTimeout = 200 * time.Millisecond
t.Cleanup(func() { lifecycleReadyTimeout = prev })
stuckAddr := startFront(t, &testFront{key: frontKey, block: true})
goodAddr := startFront(t, &testFront{key: frontKey})
start := time.Now()
conn, got, skipped := newTestHandler().selectS3Front(context.Background(), []string{stuckAddr, goodAddr})
if conn == nil {
t.Fatalf("expected the second front, skipped: %v", skipped)
}
conn.Close()
if elapsed := time.Since(start); elapsed > 5*time.Second {
t.Fatalf("readiness check took %v", elapsed)
}
if got != goodAddr || len(skipped) != 1 || status.Code(errors.Unwrap(skipped[0])) != codes.DeadlineExceeded {
t.Fatalf("got %q skipped %v, want %q after a DeadlineExceeded skip", got, skipped, goodAddr)
}
}
func TestSelectS3Front_ParentCancelStops(t *testing.T) {
setWorkerKey(t, frontKey)
next := &testFront{key: frontKey}
stuckAddr := startFront(t, &testFront{key: frontKey, block: true})
nextAddr := startFront(t, next)
ctx, cancel := context.WithCancel(context.Background())
time.AfterFunc(100*time.Millisecond, cancel)
start := time.Now()
conn, _, skipped := newTestHandler().selectS3Front(ctx, []string{stuckAddr, nextAddr})
if conn != nil {
conn.Close()
t.Fatal("selected a front after the parent was cancelled")
}
if elapsed := time.Since(start); elapsed > 5*time.Second {
t.Fatalf("cancelled selection took %v", elapsed)
}
if !errors.Is(noReadyS3Error(skipped), context.Canceled) {
t.Fatalf("want context.Canceled, got %v", skipped)
}
if next.readies.Load() != 0 {
t.Fatal("tried another front after the parent was cancelled")
}
}
func TestExecute_RejectedCredentialsFailBeforeListing(t *testing.T) {
setWorkerKey(t, otherKey)
filer := &countingFiler{}
filerAddr := serveGrpc(t, func(s *grpc.Server) { filer_pb.RegisterSeaweedFilerServer(s, filer) })
front := &testFront{key: frontKey}
frontAddr := startFront(t, front)
err := newTestHandler().Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
Job: &plugin_pb.JobSpec{
JobId: "j1", JobType: jobType,
Parameters: map[string]*plugin_pb.ConfigValue{
"filer_grpc_address": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: filerAddr}},
},
},
ClusterContext: &plugin_pb.ClusterContext{S3GrpcAddresses: []string{frontAddr}},
}, &recordingExecSender{})
if err == nil || !strings.Contains(err.Error(), "jwt.filer_signing.key") {
t.Fatalf("want a signing-key error, got %v", err)
}
assertNoSecrets(t, err.Error())
if filer.lists.Load() != 0 || front.deletes.Load() != 0 {
t.Fatalf("pass started anyway: %d listings, %d deletes", filer.lists.Load(), front.deletes.Load())
}
}