mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
* security: scoped read-only token for filer metadata subscriptions A token signed with jwt.filer_signing.subscribe.key names the subscribe methods and the path prefixes its holder may see. mTLS peers and admin tokens stay unrestricted. Internal subscribers attach an admin token when both keys are configured. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: enforce subscription token scope on metadata streams With jwt.filer_signing.subscribe.key set, SubscribeMetadata and SubscribeLocalMetadata authorize the caller first. A scoped caller gets a stream that sends only events whose paths are all under its prefixes, filters batches, and never carries log file refs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: send an admin token on internal metadata subscriptions Peer filers, metadata followers and the lifecycle reader attach an admin token when the subscribe key is configured, so they keep the full stream without mTLS. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * shell: add fs.meta.token to mint metadata subscription tokens Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: end scoped metadata subscriptions at token expiry The scope now carries the token's exp. The scoped stream's context ends at that time and the call returns Unauthenticated, so an open stream no longer outlives its token. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer.sync: sign metadata subscriptions with the source cluster's key MetadataFollowOption takes a FilerJwtProvider; filer.sync passes the one loaded from -a.security or -b.security, so clusters with different admin keys both accept the subscription. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * shell: report fs.meta.token flag errors Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: give scoped metadata subscribers their own client id A scoped caller could reuse another subscriber's ClientId with a higher epoch and replace that registration, disconnecting it. Scoped streams now register under a server-chosen id that no other subscriber holds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * security: accept only HS256 metadata subscription tokens Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * security: test HS384 metadata subscription token rejection Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: never let a scoped subscriber replace another registration The server-chosen id can be taken by another subscriber between choosing and registering it. Registering scoped streams with the lowest epoch makes addClient refuse them rather than replace the other registration. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: send position markers for events a scoped token withholds Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test a scoped stream withholding a whole event batch Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: filter SendMsg on scoped metadata streams The scoped wrappers only overrode Send, so a SendMsg on the embedded stream would reach the client unfiltered. Route it through the same filter and refuse any other message type. Ported from #12790. Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test a scoped subscription through the real subscribe loop Run SubscribeLocalMetadata with a scoped token over a flushed backlog, the in-memory tail and live events: only in-scope events arrive, sibling, "..", encoded and cross-boundary rename paths are dropped, and no log file references are sent although the client asks for them. Without the subscribe key every event still arrives. Ported from #12790. Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test that the HTTP API refuses metadata subscription tokens GET, HEAD, PUT, POST and DELETE all refuse a subscription token, also when the subscribe key equals the read or write key. Ported from #12790. Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com>
340 lines
11 KiB
Go
340 lines
11 KiB
Go
package reader
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
|
"github.com/seaweedfs/seaweedfs/weed/security"
|
|
)
|
|
|
|
// Event is one in-shard meta-log event delivered to the router.
|
|
//
|
|
// BootstrapVersion is set only by the bucket bootstrapper when it
|
|
// expands a .versions/ directory; the meta-log path leaves it nil.
|
|
// Carries pre-computed sibling state so the router can fire
|
|
// NoncurrentDays / NewerNoncurrent without listing again.
|
|
type Event struct {
|
|
TsNs int64
|
|
Bucket string
|
|
Key string
|
|
ShardID int
|
|
OldEntry *filer_pb.Entry
|
|
NewEntry *filer_pb.Entry
|
|
NewParent string
|
|
BootstrapVersion *BootstrapVersion
|
|
}
|
|
|
|
// BootstrapVersion is the per-version state computed once per
|
|
// .versions/<key>/ directory at bootstrap time. Key fields shape
|
|
// EvaluateAction: IsLatest gates current vs. noncurrent rules,
|
|
// NoncurrentIndex gates NewerNoncurrentVersions retention,
|
|
// SuccessorModTime sets the noncurrent clock (when this version was
|
|
// replaced).
|
|
type BootstrapVersion struct {
|
|
LogicalKey string
|
|
VersionID string
|
|
IsLatest bool
|
|
IsDeleteMarker bool
|
|
NumVersions int
|
|
NoncurrentIndex int // 0 = newest noncurrent
|
|
SuccessorModTime time.Time
|
|
}
|
|
|
|
// IsDelete reports whether this event removes an entry.
|
|
func (e *Event) IsDelete() bool {
|
|
return e.NewEntry == nil && e.OldEntry != nil
|
|
}
|
|
|
|
// IsCreate reports whether this event creates an entry.
|
|
func (e *Event) IsCreate() bool {
|
|
return e.OldEntry == nil && e.NewEntry != nil
|
|
}
|
|
|
|
// Reader subscribes to the filer meta-log and emits in-range Events to a
|
|
// channel. One subscription handles a contiguous span (or arbitrary set)
|
|
// of shards via ShardPredicate; the downstream router/dispatcher consume
|
|
// events and ack-advance the per-shard cursor for matched ActionKeys
|
|
// when their actions complete.
|
|
type Reader struct {
|
|
// ShardID and ShardPredicate are alternatives — set at most one.
|
|
// ShardPredicate wins if both are populated.
|
|
ShardID int // [0, s3lifecycle.ShardCount); used when ShardPredicate is nil
|
|
ShardPredicate func(int) bool // accepts an event when true; nil falls back to ShardID equality
|
|
|
|
BucketsPath string // e.g. "/buckets"
|
|
// Cursor is the single-shard cursor used for SinceNs when StartTsNs is 0.
|
|
// Range callers pass StartTsNs directly and leave Cursor nil; SinceNs=0
|
|
// then means "subscribe from the start of the meta-log".
|
|
Cursor *Cursor
|
|
StartTsNs int64
|
|
Events chan<- *Event
|
|
|
|
// UntilTsNs ends the stream once the filer has delivered through this
|
|
// timestamp. Zero follows forever, which on an idle cluster parks a
|
|
// bounded caller in Recv until something unrelated is written.
|
|
UntilTsNs int64
|
|
|
|
// EventBudget caps how many events Run processes before returning nil.
|
|
// Zero = unbounded; the run continues until ctx cancellation or stream
|
|
// error. Used by the worker scheduler to bound a single READ task.
|
|
EventBudget int
|
|
|
|
// ReceiveTimeout bounds the wait for each response, covering a filer
|
|
// that stops producing while the transport still answers keepalives.
|
|
// Also opts into idle heartbeats so a caught-up stream stays alive.
|
|
// Keep above the filer's 15m maxGapStall — a subscriber parked on a
|
|
// gap sends nothing and is not stuck. Zero disables it.
|
|
ReceiveTimeout time.Duration
|
|
|
|
// bucketsPathSlash is BucketsPath with a guaranteed trailing slash,
|
|
// computed once on Run and reused per event to avoid recomputing the
|
|
// normalized prefix in extractBucketKey.
|
|
bucketsPathSlash string
|
|
}
|
|
|
|
// ErrReceiveTimeout: stream still open, but no events and no heartbeats.
|
|
var ErrReceiveTimeout = errors.New("reader: metadata receive timeout")
|
|
|
|
type receiveResult struct {
|
|
resp *filer_pb.SubscribeMetadataResponse
|
|
err error
|
|
}
|
|
|
|
// awaitResponse waits for the next response under ReceiveTimeout. Started
|
|
// per call, so it times the filer only — a slow Events consumer blocks in
|
|
// dispatchOne, outside this window, and can't trip the watchdog.
|
|
func (r *Reader) awaitResponse(ctx context.Context, received <-chan receiveResult) (*filer_pb.SubscribeMetadataResponse, error, bool) {
|
|
var timeout <-chan time.Time
|
|
if r.ReceiveTimeout > 0 {
|
|
timer := time.NewTimer(r.ReceiveTimeout)
|
|
defer timer.Stop()
|
|
timeout = timer.C
|
|
}
|
|
select {
|
|
case result := <-received:
|
|
return result.resp, result.err, false
|
|
case <-timeout:
|
|
return nil, nil, true
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err(), false
|
|
}
|
|
}
|
|
|
|
// Run subscribes via SubscribeMetadata starting at the configured position,
|
|
// filters to the configured shard set, and emits Events. Returns on
|
|
// ctx.Done(), io.EOF, or stream error. Caller is responsible for closing
|
|
// Events if it owns it.
|
|
func (r *Reader) Run(ctx context.Context, client filer_pb.SeaweedFilerClient, clientName string, clientID int32) error {
|
|
if r.ShardPredicate == nil {
|
|
if r.ShardID < 0 || r.ShardID >= s3lifecycle.ShardCount {
|
|
return fmt.Errorf("reader: shard_id %d out of range and no ShardPredicate", r.ShardID)
|
|
}
|
|
}
|
|
if r.Events == nil {
|
|
return errors.New("reader: nil Events channel")
|
|
}
|
|
if r.BucketsPath == "" {
|
|
return errors.New("reader: empty BucketsPath")
|
|
}
|
|
if r.ReceiveTimeout < 0 {
|
|
return fmt.Errorf("reader: negative ReceiveTimeout %v", r.ReceiveTimeout)
|
|
}
|
|
r.bucketsPathSlash = r.BucketsPath
|
|
if !strings.HasSuffix(r.bucketsPathSlash, "/") {
|
|
r.bucketsPathSlash += "/"
|
|
}
|
|
|
|
sinceNs := r.StartTsNs
|
|
if sinceNs == 0 && r.Cursor != nil {
|
|
sinceNs = r.Cursor.MinTsNs()
|
|
}
|
|
// Own context: aborting the RPC is the only way to unblock Recv.
|
|
streamCtx, cancelStream := context.WithCancel(ctx)
|
|
defer cancelStream()
|
|
stream, err := client.SubscribeMetadata(security.WithMetadataSubscriptionCredentials(streamCtx, nil), &filer_pb.SubscribeMetadataRequest{
|
|
ClientName: clientName,
|
|
PathPrefix: r.BucketsPath,
|
|
SinceNs: sinceNs,
|
|
UntilNs: r.UntilTsNs,
|
|
ClientId: clientID,
|
|
ClientSupportsBatching: true,
|
|
// dispatchOne drops these, but arriving at all is the point.
|
|
ClientSupportsIdleHeartbeat: r.ReceiveTimeout > 0,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("subscribe: %w", err)
|
|
}
|
|
|
|
// Buffered so a response landing as the watchdog fires doesn't strand
|
|
// this goroutine.
|
|
received := make(chan receiveResult, 1)
|
|
go func() {
|
|
for {
|
|
resp, recvErr := stream.Recv()
|
|
select {
|
|
case received <- receiveResult{resp: resp, err: recvErr}:
|
|
case <-streamCtx.Done():
|
|
return
|
|
}
|
|
if recvErr != nil {
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
|
|
processed := 0
|
|
for {
|
|
resp, recvErr, timedOut := r.awaitResponse(streamCtx, received)
|
|
if timedOut {
|
|
cancelStream() // unwind the goroutine blocked in Recv
|
|
return fmt.Errorf("%w after %s", ErrReceiveTimeout, r.ReceiveTimeout)
|
|
}
|
|
if recvErr == io.EOF {
|
|
return nil
|
|
}
|
|
if recvErr != nil {
|
|
return recvErr
|
|
}
|
|
|
|
// First event in resp is the primary; resp.Events is the batched tail.
|
|
if err := r.dispatchOne(ctx, resp, &processed); err != nil {
|
|
return err
|
|
}
|
|
for _, ev := range resp.Events {
|
|
if err := r.dispatchOne(ctx, ev, &processed); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if r.EventBudget > 0 && processed >= r.EventBudget {
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *Reader) dispatchOne(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, processed *int) error {
|
|
if resp == nil || resp.EventNotification == nil {
|
|
return nil
|
|
}
|
|
bucket, key, ok := r.extractBucketKey(resp)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
shardID := s3lifecycle.ShardID(bucket, key)
|
|
if r.ShardPredicate != nil {
|
|
if !r.ShardPredicate(shardID) {
|
|
return nil
|
|
}
|
|
} else if shardID != r.ShardID {
|
|
return nil
|
|
}
|
|
|
|
ev := &Event{
|
|
TsNs: resp.TsNs,
|
|
Bucket: bucket,
|
|
Key: key,
|
|
ShardID: shardID,
|
|
OldEntry: resp.EventNotification.OldEntry,
|
|
NewEntry: resp.EventNotification.NewEntry,
|
|
NewParent: resp.EventNotification.NewParentPath,
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case r.Events <- ev:
|
|
*processed++
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// extractBucketKey turns a meta-log event's path into (bucket, key) when the
|
|
// event lies under BucketsPath. Returns ok=false for events outside that
|
|
// subtree (cluster admin entries, system files, etc.) so the reader can skip
|
|
// them without engaging the routing index.
|
|
//
|
|
// The path is reconstructed as DirectoryPath/Name, where DirectoryPath comes
|
|
// from the entry context and Name from old_entry/new_entry. We prefer
|
|
// new_entry on creates/updates and old_entry on deletes; both carry the same
|
|
// Name on renames where new_parent_path differs.
|
|
func (r *Reader) extractBucketKey(resp *filer_pb.SubscribeMetadataResponse) (string, string, bool) {
|
|
notif := resp.EventNotification
|
|
dir := notif.NewParentPath
|
|
if dir == "" {
|
|
// On deletes, NewParentPath may be empty; the directory is encoded
|
|
// in resp.Directory.
|
|
dir = resp.Directory
|
|
}
|
|
var name string
|
|
switch {
|
|
case notif.NewEntry != nil:
|
|
name = notif.NewEntry.Name
|
|
case notif.OldEntry != nil:
|
|
name = notif.OldEntry.Name
|
|
default:
|
|
return "", "", false
|
|
}
|
|
|
|
// Pre-normalized prefix (BucketsPath with trailing slash) is computed
|
|
// once in Run; bucket-root events arrive as either "/buckets" or
|
|
// "/buckets/", so accept both. The fallback path mirrors Run's
|
|
// normalization for tests that call extractBucketKey directly.
|
|
prefix := r.bucketsPathSlash
|
|
if prefix == "" {
|
|
prefix = r.BucketsPath
|
|
if !strings.HasSuffix(prefix, "/") {
|
|
prefix += "/"
|
|
}
|
|
}
|
|
bare := strings.TrimSuffix(prefix, "/")
|
|
var rest string
|
|
switch {
|
|
case dir == bare || dir == prefix:
|
|
// Bucket create/delete at /buckets root: bucket name is the entry name.
|
|
if name == "" {
|
|
return "", "", false
|
|
}
|
|
return name, "", true
|
|
case strings.HasPrefix(dir, prefix):
|
|
rest = dir[len(prefix):]
|
|
default:
|
|
return "", "", false
|
|
}
|
|
// rest = "<bucket>" or "<bucket>/<sub>/<sub>..."
|
|
slash := strings.IndexByte(rest, '/')
|
|
var bucket, parentInBucket string
|
|
if slash < 0 {
|
|
bucket = rest
|
|
} else {
|
|
bucket = rest[:slash]
|
|
parentInBucket = rest[slash+1:]
|
|
}
|
|
if bucket == "" {
|
|
return "", "", false
|
|
}
|
|
if parentInBucket != "" {
|
|
return bucket, parentInBucket + "/" + name, true
|
|
}
|
|
return bucket, name, true
|
|
}
|
|
|
|
// LogStartup is a small helper for callers that want a one-line readable
|
|
// description of where the reader is starting.
|
|
func (r *Reader) LogStartup() {
|
|
sinceNs := r.StartTsNs
|
|
if sinceNs == 0 && r.Cursor != nil {
|
|
sinceNs = r.Cursor.MinTsNs()
|
|
}
|
|
if r.ShardPredicate != nil {
|
|
glog.V(1).Infof("lifecycle reader: shard=range sinceNs=%d budget=%d", sinceNs, r.EventBudget)
|
|
return
|
|
}
|
|
glog.V(1).Infof("lifecycle reader: shard=%d sinceNs=%d budget=%d",
|
|
r.ShardID, sinceNs, r.EventBudget)
|
|
}
|