mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-06 12:07:51 +01:00
feat(s3/lifecycle): delete streaming algorithm path (Phase 5b) (#9466)
* feat(s3/lifecycle): delete streaming algorithm path (Phase 5b) Phase 5a (PR #9465) retired the algorithm flag and made daily_replay the only execution path. The streaming-side code (scheduler.Scheduler, scheduler.BucketBootstrapper, dispatcher.Pipeline, dispatcher.Dispatcher, dispatcher.FilerPersister, and their tests) has had no in-tree caller since then. This PR deletes it. Net change: ~4800 lines removed, ~130 added (the scheduler/configload tests' helper file the deleted bootstrap_test.go used to host). Removed: - weed/s3api/s3lifecycle/scheduler/{bootstrap,bootstrap_test, scheduler,scheduler_test,pipeline_fanout_test, refresh_default,refresh_s3tests}.go - weed/s3api/s3lifecycle/dispatcher/{dispatcher,dispatcher_test, dispatcher_helpers_test,edge_cases_test,multi_shard_test, pipeline,pipeline_test,pipeline_helpers_test,toproto_test, dispatch_ticks_default,dispatch_ticks_s3tests}.go - weed/s3api/s3lifecycle/dispatcher/filer_persister_test.go (FilerPersister deleted; FilerStore tests don't need their own file) - weed/shell/command_s3_lifecycle_run_shard{,_test}.go (debug-only shell command that only ever wrapped the streaming pipeline; the production worker now exercises the same path every daily run) Trimmed: - dispatcher/filer_persister.go down to FilerStore + NewFilerStoreClient — the small interface daily_replay's cursor persister (dailyrun.FilerCursorPersister) plugs into. Kept (still consumed by daily_replay): - scheduler/configload.{go,_test.go} (LoadCompileInputs, AllActivePriorStates) - dispatcher/sibling_lister.{go,_test.go} (NewFilerSiblingLister, FilerSiblingLister) - dispatcher/filer_persister.go (FilerStore, NewFilerStoreClient) scheduler/testhelpers_test.go restores fakeFilerClient, fakeListStream, dirEntry, fileEntry — helpers the configload tests used to share with the deleted bootstrap_test.go. Updates the handler-package doc strings and one reader-package comment that still named the streaming pipeline. * fix(s3/lifecycle): hold lock through tree read in test filer client gemini caught an inconsistency in scheduler/testhelpers_test.go: LookupDirectoryEntry reads c.tree under c.mu, but ListEntries was releasing the lock before reading c.tree. The map is effectively static during tests so there's no actual race today, but matching the convention keeps the helper safe if a future test mutates the tree mid-run.
This commit is contained in:
Chris Lu authored and GitHub committed
1 parent
745e864bda
commit
5004b4e542
25 files changed
+131
-4804
No files matched your search
@@ -1,329 +0,0 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/dispatcher"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/scheduler"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func init() {
|
||||
Commands = append(Commands, &commandS3LifecycleRunShard{})
|
||||
}
|
||||
|
||||
type commandS3LifecycleRunShard struct{}
|
||||
|
||||
func (c *commandS3LifecycleRunShard) Name() string {
|
||||
return "s3.lifecycle.run-shard"
|
||||
}
|
||||
|
||||
func (c *commandS3LifecycleRunShard) Help() string {
|
||||
return `manually run one or more shards of the event-driven S3 lifecycle worker
|
||||
|
||||
Subscribes once to the filer meta-log, filters events to the configured
|
||||
(bucket, key-prefix-hash) shards, routes them through the compiled lifecycle
|
||||
engine, and dispatches due actions to the S3 server's LifecycleDelete RPC.
|
||||
Persists each shard's cursor to /etc/s3/lifecycle/cursors/shard-NN.json so
|
||||
subsequent runs resume.
|
||||
|
||||
The -shards form covers a range or set; one filer subscription handles the
|
||||
whole set, with no per-shard goroutine fan-out. Provide either -shard or
|
||||
-shards, not both.
|
||||
|
||||
# single shard
|
||||
s3.lifecycle.run-shard -shard 0 -s3 localhost:8333 -events 100
|
||||
|
||||
# contiguous range, all 16 shards via one subscription
|
||||
s3.lifecycle.run-shard -shards 0-15 -s3 localhost:8333 -events 5000
|
||||
|
||||
# explicit set
|
||||
s3.lifecycle.run-shard -shards 0,3,7 -s3 localhost:8333
|
||||
|
||||
# custom cadence
|
||||
s3.lifecycle.run-shard -shards 0-15 -s3 s3-host:8333 -dispatch 1s -checkpoint 10s
|
||||
`
|
||||
}
|
||||
|
||||
func (c *commandS3LifecycleRunShard) HasTag(CommandTag) bool { return false }
|
||||
|
||||
func (c *commandS3LifecycleRunShard) Do(args []string, env *CommandEnv, writer io.Writer) error {
|
||||
fs := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
|
||||
shard := fs.Int("shard", -1, "single shard id in [0, 16); use -shards for a range or set")
|
||||
shardsSpec := fs.String("shards", "", "shard range \"lo-hi\" or comma list \"a,b,c\"; mutually exclusive with -shard")
|
||||
s3Endpoint := fs.String("s3", "", "s3 server gRPC endpoint, host:port")
|
||||
eventBudget := fs.Int("events", 1000, "max in-shard events to process before returning (0 = unbounded; counts only events that pass the shard filter)")
|
||||
dispatchTick := fs.Duration("dispatch", 5*time.Second, "dispatcher tick cadence")
|
||||
checkpointTick := fs.Duration("checkpoint", 30*time.Second, "cursor checkpoint cadence")
|
||||
refreshInterval := fs.Duration("refresh", 5*time.Minute, "interval for rebuilding the engine snapshot from filer-backed bucket configs; 0 = compile once at startup")
|
||||
bootstrapInterval := fs.Duration("bootstrap-interval", 0, "cadence for revisiting each bucket's bootstrap walk; 0 = walk once per process. scan_only actions only fire from bootstrap, so a long-running worker needs a non-zero value to handle their retention horizon")
|
||||
runtime := fs.Duration("runtime", 0, "wall-clock cap on the run; 0 = no timeout. -events alone can hang on quiet shards")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
shards, err := resolveShardSelection(*shard, *shardsSpec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *s3Endpoint == "" {
|
||||
return fmt.Errorf("-s3 required (host:port of s3 server gRPC)")
|
||||
}
|
||||
if *eventBudget < 0 {
|
||||
return fmt.Errorf("-events must be >= 0 (0 = unbounded)")
|
||||
}
|
||||
|
||||
bucketsPath, err := resolveBucketsPath(env)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve buckets path: %w", err)
|
||||
}
|
||||
fmt.Fprintf(writer, "buckets path: %s\n", bucketsPath)
|
||||
|
||||
dialCtx, dialCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
conn, err := pb.GrpcDial(dialCtx, *s3Endpoint, false, env.option.GrpcDialOption)
|
||||
dialCancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("dial s3 %s: %w", *s3Endpoint, err)
|
||||
}
|
||||
defer conn.Close()
|
||||
rpcClient := s3_lifecycle_pb.NewSeaweedS3LifecycleInternalClient(conn)
|
||||
|
||||
// Run the whole pipeline inside one WithFilerClient so the reader's
|
||||
// SubscribeMetadata stream and the persister share a single connection.
|
||||
return env.WithFilerClient(true, func(filerClient filer_pb.SeaweedFilerClient) error {
|
||||
eng := engine.New()
|
||||
|
||||
pipeline := &dispatcher.Pipeline{
|
||||
Shards: shards,
|
||||
BucketsPath: bucketsPath,
|
||||
Engine: eng,
|
||||
Persister: &dispatcher.FilerPersister{Store: dispatcher.NewFilerStoreClient(filerClient)},
|
||||
Client: &lifecycleClientCallable{c: rpcClient},
|
||||
FilerClient: filerClient,
|
||||
ClientID: util.RandomInt32(),
|
||||
ClientName: fmt.Sprintf("shell-lifecycle-%s", formatShardLabel(shards)),
|
||||
DispatchTick: *dispatchTick,
|
||||
CheckpointTick: *checkpointTick,
|
||||
EventBudget: *eventBudget,
|
||||
}
|
||||
|
||||
bsr := &scheduler.BucketBootstrapper{
|
||||
FilerClient: filerClient,
|
||||
BucketsPath: bucketsPath,
|
||||
Injector: pipeline,
|
||||
BootstrapInterval: *bootstrapInterval,
|
||||
}
|
||||
bootstrapCtx, bootstrapCancel := context.WithCancel(context.Background())
|
||||
defer bootstrapCancel()
|
||||
|
||||
compile := func(initial bool) {
|
||||
inputs, parseErrors, err := scheduler.LoadCompileInputs(context.Background(), filerClient, bucketsPath)
|
||||
if err != nil {
|
||||
if initial {
|
||||
fmt.Fprintf(writer, "warning: load lifecycle configs: %v\n", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if initial {
|
||||
for i, pe := range parseErrors {
|
||||
if i < 3 {
|
||||
fmt.Fprintf(writer, "warning: %s: %v\n", pe.Bucket, pe.Err)
|
||||
}
|
||||
}
|
||||
if extra := len(parseErrors) - 3; extra > 0 {
|
||||
fmt.Fprintf(writer, "warning: %d additional bucket(s) had malformed lifecycle config\n", extra)
|
||||
}
|
||||
if len(inputs) == 0 {
|
||||
fmt.Fprintln(writer, "no buckets with enabled lifecycle rules at startup; will refresh and pick them up as they're added")
|
||||
} else {
|
||||
fmt.Fprintf(writer, "loaded lifecycle for %d bucket(s)\n", len(inputs))
|
||||
}
|
||||
}
|
||||
eng.Compile(inputs, engine.CompileOptions{PriorStates: scheduler.AllActivePriorStates(inputs)})
|
||||
// First time we see a bucket, spin up a one-shot walker that
|
||||
// lists existing entries and synthesizes events. The reader-
|
||||
// driven path only sees events created after the rule lands,
|
||||
// so without this backfill objects PUT before the rule (the
|
||||
// s3-tests scenario) would never expire.
|
||||
buckets := make([]string, 0, len(inputs))
|
||||
for _, in := range inputs {
|
||||
buckets = append(buckets, in.Bucket)
|
||||
}
|
||||
bsr.KickOffNew(bootstrapCtx, buckets)
|
||||
}
|
||||
compile(true)
|
||||
|
||||
var ctx context.Context
|
||||
var cancel context.CancelFunc
|
||||
if *runtime > 0 {
|
||||
ctx, cancel = context.WithTimeout(context.Background(), *runtime)
|
||||
} else {
|
||||
ctx, cancel = context.WithCancel(context.Background())
|
||||
}
|
||||
defer cancel()
|
||||
|
||||
// Periodic engine rebuild so rules added (or disabled) after startup
|
||||
// land in the snapshot the dispatcher reads on its next tick.
|
||||
if *refreshInterval > 0 {
|
||||
go func() {
|
||||
t := time.NewTicker(*refreshInterval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
compile(false)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
fmt.Fprintf(writer, "running shards %s (event budget=%d, runtime=%s, refresh=%s)…\n", formatShardLabel(shards), *eventBudget, *runtime, *refreshInterval)
|
||||
if err := pipeline.Run(ctx); err != nil {
|
||||
return fmt.Errorf("pipeline: %w", err)
|
||||
}
|
||||
fmt.Fprintf(writer, "shards %s complete; cursors checkpointed\n", formatShardLabel(shards))
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// resolveShardSelection turns the -shard / -shards flags into a sorted,
|
||||
// deduplicated []int. Exactly one form must be specified.
|
||||
func resolveShardSelection(singleShard int, shardsSpec string) ([]int, error) {
|
||||
if singleShard >= 0 && shardsSpec != "" {
|
||||
return nil, fmt.Errorf("-shard and -shards are mutually exclusive")
|
||||
}
|
||||
if singleShard < 0 && shardsSpec == "" {
|
||||
return nil, fmt.Errorf("specify -shard <id> or -shards <range|set>")
|
||||
}
|
||||
if singleShard >= 0 {
|
||||
if singleShard >= s3lifecycle.ShardCount {
|
||||
return nil, fmt.Errorf("-shard %d out of [0,%d)", singleShard, s3lifecycle.ShardCount)
|
||||
}
|
||||
return []int{singleShard}, nil
|
||||
}
|
||||
return parseShardsSpec(shardsSpec)
|
||||
}
|
||||
|
||||
// parseShardsSpec accepts "lo-hi" (inclusive) or "a,b,c" and returns a
|
||||
// sorted, deduplicated, in-range []int.
|
||||
func parseShardsSpec(spec string) ([]int, error) {
|
||||
spec = strings.TrimSpace(spec)
|
||||
seen := map[int]struct{}{}
|
||||
add := func(v int) error {
|
||||
if v < 0 || v >= s3lifecycle.ShardCount {
|
||||
return fmt.Errorf("shard %d out of [0,%d)", v, s3lifecycle.ShardCount)
|
||||
}
|
||||
seen[v] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
if strings.Contains(spec, "-") && !strings.Contains(spec, ",") {
|
||||
parts := strings.SplitN(spec, "-", 2)
|
||||
lo, err := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("range lo: %w", err)
|
||||
}
|
||||
hi, err := strconv.Atoi(strings.TrimSpace(parts[1]))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("range hi: %w", err)
|
||||
}
|
||||
if lo > hi {
|
||||
return nil, fmt.Errorf("range lo %d > hi %d", lo, hi)
|
||||
}
|
||||
for v := lo; v <= hi; v++ {
|
||||
if err := add(v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for _, part := range strings.Split(spec, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
v, err := strconv.Atoi(part)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("shard list: %w", err)
|
||||
}
|
||||
if err := add(v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(seen) == 0 {
|
||||
return nil, fmt.Errorf("empty shard set")
|
||||
}
|
||||
out := make([]int, 0, len(seen))
|
||||
for v := range seen {
|
||||
out = append(out, v)
|
||||
}
|
||||
sort.Ints(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func formatShardLabel(shards []int) string {
|
||||
if len(shards) == 1 {
|
||||
return fmt.Sprintf("%d", shards[0])
|
||||
}
|
||||
// Detect contiguous range.
|
||||
contiguous := true
|
||||
for i := 1; i < len(shards); i++ {
|
||||
if shards[i] != shards[i-1]+1 {
|
||||
contiguous = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if contiguous {
|
||||
return fmt.Sprintf("%d-%d", shards[0], shards[len(shards)-1])
|
||||
}
|
||||
parts := make([]string, len(shards))
|
||||
for i, v := range shards {
|
||||
parts[i] = strconv.Itoa(v)
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
// lifecycleClientCallable adapts the generated grpc client (variadic
|
||||
// CallOption tail) to the dispatcher.LifecycleClient interface.
|
||||
type lifecycleClientCallable struct {
|
||||
c s3_lifecycle_pb.SeaweedS3LifecycleInternalClient
|
||||
}
|
||||
|
||||
func (l *lifecycleClientCallable) LifecycleDelete(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
|
||||
return l.c.LifecycleDelete(ctx, req)
|
||||
}
|
||||
|
||||
// resolveBucketsPath fetches the filer's configured buckets directory.
|
||||
// Falls back to /buckets when the filer doesn't return one.
|
||||
func resolveBucketsPath(env *CommandEnv) (string, error) {
|
||||
var path string
|
||||
err := env.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
path = resp.GetDirBuckets()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if path == "" {
|
||||
path = "/buckets"
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseShardsSpec_Range(t *testing.T) {
|
||||
got, err := parseShardsSpec("3-7")
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
want := []int{3, 4, 5, 6, 7}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseShardsSpec_Set(t *testing.T) {
|
||||
got, err := parseShardsSpec("0,3,7")
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
want := []int{0, 3, 7}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseShardsSpec_DedupSort(t *testing.T) {
|
||||
got, err := parseShardsSpec("7,3,3,0")
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
want := []int{0, 3, 7}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseShardsSpec_OutOfRange(t *testing.T) {
|
||||
if _, err := parseShardsSpec("16"); err == nil {
|
||||
t.Fatal("expected out-of-range error for 16")
|
||||
}
|
||||
if _, err := parseShardsSpec("-1"); err == nil {
|
||||
t.Fatal("expected out-of-range error for -1")
|
||||
}
|
||||
if _, err := parseShardsSpec("0-16"); err == nil {
|
||||
t.Fatal("expected out-of-range error for 0-16")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseShardsSpec_BadRange(t *testing.T) {
|
||||
if _, err := parseShardsSpec("7-3"); err == nil {
|
||||
t.Fatal("expected lo>hi error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveShardSelection_Mutex(t *testing.T) {
|
||||
if _, err := resolveShardSelection(0, "1,2"); err == nil {
|
||||
t.Fatal("expected mutex error when both -shard and -shards set")
|
||||
}
|
||||
if _, err := resolveShardSelection(-1, ""); err == nil {
|
||||
t.Fatal("expected error when neither set")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveShardSelection_SingleShard(t *testing.T) {
|
||||
got, err := resolveShardSelection(5, "")
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, []int{5}) {
|
||||
t.Fatalf("got %v, want [5]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatShardLabel(t *testing.T) {
|
||||
cases := []struct {
|
||||
in []int
|
||||
want string
|
||||
}{
|
||||
{[]int{5}, "5"},
|
||||
{[]int{0, 1, 2, 3}, "0-3"},
|
||||
{[]int{0, 2, 5}, "0,2,5"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := formatShardLabel(tc.in); got != tc.want {
|
||||
t.Errorf("formatShardLabel(%v)=%q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user