mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-12 17:40:43 +02:00
* feat(s3/lifecycle): delete streaming algorithm path (Phase 5b) Phase 5a (PR #9465) retired the algorithm flag and made daily_replay the only execution path. The streaming-side code (scheduler.Scheduler, scheduler.BucketBootstrapper, dispatcher.Pipeline, dispatcher.Dispatcher, dispatcher.FilerPersister, and their tests) has had no in-tree caller since then. This PR deletes it. Net change: ~4800 lines removed, ~130 added (the scheduler/configload tests' helper file the deleted bootstrap_test.go used to host). Removed: - weed/s3api/s3lifecycle/scheduler/{bootstrap,bootstrap_test, scheduler,scheduler_test,pipeline_fanout_test, refresh_default,refresh_s3tests}.go - weed/s3api/s3lifecycle/dispatcher/{dispatcher,dispatcher_test, dispatcher_helpers_test,edge_cases_test,multi_shard_test, pipeline,pipeline_test,pipeline_helpers_test,toproto_test, dispatch_ticks_default,dispatch_ticks_s3tests}.go - weed/s3api/s3lifecycle/dispatcher/filer_persister_test.go (FilerPersister deleted; FilerStore tests don't need their own file) - weed/shell/command_s3_lifecycle_run_shard{,_test}.go (debug-only shell command that only ever wrapped the streaming pipeline; the production worker now exercises the same path every daily run) Trimmed: - dispatcher/filer_persister.go down to FilerStore + NewFilerStoreClient — the small interface daily_replay's cursor persister (dailyrun.FilerCursorPersister) plugs into. Kept (still consumed by daily_replay): - scheduler/configload.{go,_test.go} (LoadCompileInputs, AllActivePriorStates) - dispatcher/sibling_lister.{go,_test.go} (NewFilerSiblingLister, FilerSiblingLister) - dispatcher/filer_persister.go (FilerStore, NewFilerStoreClient) scheduler/testhelpers_test.go restores fakeFilerClient, fakeListStream, dirEntry, fileEntry — helpers the configload tests used to share with the deleted bootstrap_test.go. Updates the handler-package doc strings and one reader-package comment that still named the streaming pipeline. * fix(s3/lifecycle): hold lock through tree read in test filer client gemini caught an inconsistency in scheduler/testhelpers_test.go: LookupDirectoryEntry reads c.tree under c.mu, but ListEntries was releasing the lock before reading c.tree. The map is effectively static during tests so there's no actual race today, but matching the convention keeps the helper safe if a future test mutates the tree mid-run.
120 lines
3.1 KiB
Go
120 lines
3.1 KiB
Go
package scheduler
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
"sort"
|
|
"sync"
|
|
"sync/atomic"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/metadata"
|
|
)
|
|
|
|
// fakeListStream implements grpc.ServerStreamingClient[filer_pb.ListEntriesResponse]
|
|
// for configload tests.
|
|
type fakeListStream struct {
|
|
responses []*filer_pb.ListEntriesResponse
|
|
index int
|
|
ctx context.Context
|
|
}
|
|
|
|
func (s *fakeListStream) Recv() (*filer_pb.ListEntriesResponse, error) {
|
|
if s.ctx != nil {
|
|
if err := s.ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if s.index >= len(s.responses) {
|
|
return nil, io.EOF
|
|
}
|
|
r := s.responses[s.index]
|
|
s.index++
|
|
return r, nil
|
|
}
|
|
|
|
func (s *fakeListStream) Header() (metadata.MD, error) { return metadata.MD{}, nil }
|
|
func (s *fakeListStream) Trailer() metadata.MD { return metadata.MD{} }
|
|
func (s *fakeListStream) CloseSend() error { return nil }
|
|
func (s *fakeListStream) Context() context.Context {
|
|
if s.ctx != nil {
|
|
return s.ctx
|
|
}
|
|
return context.Background()
|
|
}
|
|
func (s *fakeListStream) SendMsg(any) error { return nil }
|
|
func (s *fakeListStream) RecvMsg(any) error { return nil }
|
|
|
|
// fakeFilerClient is the in-memory filer used by configload tests.
|
|
type fakeFilerClient struct {
|
|
filer_pb.SeaweedFilerClient
|
|
|
|
mu sync.Mutex
|
|
tree map[string][]*filer_pb.Entry
|
|
listed []string
|
|
listedN int32
|
|
}
|
|
|
|
func (c *fakeFilerClient) LookupDirectoryEntry(_ context.Context, in *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
for _, e := range c.tree[in.Directory] {
|
|
if e != nil && e.Name == in.Name {
|
|
return &filer_pb.LookupDirectoryEntryResponse{Entry: e}, nil
|
|
}
|
|
}
|
|
return nil, filer_pb.ErrNotFound
|
|
}
|
|
|
|
func (c *fakeFilerClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, _ ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
|
|
c.mu.Lock()
|
|
c.listed = append(c.listed, in.Directory)
|
|
src := c.tree[in.Directory]
|
|
c.mu.Unlock()
|
|
atomic.AddInt32(&c.listedN, 1)
|
|
|
|
filtered := make([]*filer_pb.Entry, 0, len(src))
|
|
for _, e := range src {
|
|
if e == nil {
|
|
continue
|
|
}
|
|
if in.StartFromFileName != "" {
|
|
if in.InclusiveStartFrom {
|
|
if e.Name < in.StartFromFileName {
|
|
continue
|
|
}
|
|
} else if e.Name <= in.StartFromFileName {
|
|
continue
|
|
}
|
|
}
|
|
filtered = append(filtered, e)
|
|
}
|
|
sort.SliceStable(filtered, func(i, j int) bool { return filtered[i].Name < filtered[j].Name })
|
|
if in.Limit > 0 && uint32(len(filtered)) > in.Limit {
|
|
filtered = filtered[:in.Limit]
|
|
}
|
|
resps := make([]*filer_pb.ListEntriesResponse, 0, len(filtered))
|
|
for _, e := range filtered {
|
|
resps = append(resps, &filer_pb.ListEntriesResponse{Entry: e})
|
|
}
|
|
return &fakeListStream{responses: resps, ctx: ctx}, nil
|
|
}
|
|
|
|
func dirEntry(name string, extended map[string][]byte) *filer_pb.Entry {
|
|
return &filer_pb.Entry{
|
|
Name: name,
|
|
IsDirectory: true,
|
|
Attributes: &filer_pb.FuseAttributes{},
|
|
Extended: extended,
|
|
}
|
|
}
|
|
|
|
func fileEntry(name string) *filer_pb.Entry {
|
|
return &filer_pb.Entry{
|
|
Name: name,
|
|
IsDirectory: false,
|
|
Attributes: &filer_pb.FuseAttributes{},
|
|
}
|
|
}
|