mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
* security: scoped read-only token for filer metadata subscriptions A token signed with jwt.filer_signing.subscribe.key names the subscribe methods and the path prefixes its holder may see. mTLS peers and admin tokens stay unrestricted. Internal subscribers attach an admin token when both keys are configured. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: enforce subscription token scope on metadata streams With jwt.filer_signing.subscribe.key set, SubscribeMetadata and SubscribeLocalMetadata authorize the caller first. A scoped caller gets a stream that sends only events whose paths are all under its prefixes, filters batches, and never carries log file refs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: send an admin token on internal metadata subscriptions Peer filers, metadata followers and the lifecycle reader attach an admin token when the subscribe key is configured, so they keep the full stream without mTLS. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * shell: add fs.meta.token to mint metadata subscription tokens Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: end scoped metadata subscriptions at token expiry The scope now carries the token's exp. The scoped stream's context ends at that time and the call returns Unauthenticated, so an open stream no longer outlives its token. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer.sync: sign metadata subscriptions with the source cluster's key MetadataFollowOption takes a FilerJwtProvider; filer.sync passes the one loaded from -a.security or -b.security, so clusters with different admin keys both accept the subscription. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * shell: report fs.meta.token flag errors Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: give scoped metadata subscribers their own client id A scoped caller could reuse another subscriber's ClientId with a higher epoch and replace that registration, disconnecting it. Scoped streams now register under a server-chosen id that no other subscriber holds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * security: accept only HS256 metadata subscription tokens Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * security: test HS384 metadata subscription token rejection Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: never let a scoped subscriber replace another registration The server-chosen id can be taken by another subscriber between choosing and registering it. Registering scoped streams with the lowest epoch makes addClient refuse them rather than replace the other registration. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: send position markers for events a scoped token withholds Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test a scoped stream withholding a whole event batch Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: filter SendMsg on scoped metadata streams The scoped wrappers only overrode Send, so a SendMsg on the embedded stream would reach the client unfiltered. Route it through the same filter and refuse any other message type. Ported from #12790. Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test a scoped subscription through the real subscribe loop Run SubscribeLocalMetadata with a scoped token over a flushed backlog, the in-memory tail and live events: only in-scope events arrive, sibling, "..", encoded and cross-boundary rename paths are dropped, and no log file references are sent although the client asks for them. Without the subscribe key every event still arrives. Ported from #12790. Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test that the HTTP API refuses metadata subscription tokens GET, HEAD, PUT, POST and DELETE all refuse a subscription token, also when the subscribe key equals the read or write key. Ported from #12790. Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com>
181 lines
5.8 KiB
Go
181 lines
5.8 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"math"
|
|
"math/rand/v2"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/security"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
)
|
|
|
|
func metadataSubscriptionScope(ctx context.Context, method string) (*security.MetadataSubscriptionScope, error) {
|
|
v := util.GetViper()
|
|
return security.CheckGrpcMetadataSubscription(ctx,
|
|
security.SigningKey(v.GetString("jwt.filer_signing.subscribe.key")),
|
|
security.SigningKey(v.GetString("jwt.filer_signing.key")),
|
|
method)
|
|
}
|
|
|
|
// scopeMetadataResponse keeps only what a scoped subscriber may see: events
|
|
// whose every path is in scope, so a rename across the boundary never shows
|
|
// the far side, and no log file refs. Returns nil when nothing is left.
|
|
func scopeMetadataResponse(resp *filer_pb.SubscribeMetadataResponse, prefixes []string) *filer_pb.SubscribeMetadataResponse {
|
|
if resp.EventNotification == nil && len(resp.LogFileRefs) > 0 {
|
|
return nil
|
|
}
|
|
var kept []*filer_pb.SubscribeMetadataResponse
|
|
for _, e := range resp.Events {
|
|
if metadataEventInScope(e, prefixes) {
|
|
kept = append(kept, e)
|
|
}
|
|
}
|
|
headInScope := metadataEventInScope(resp, prefixes)
|
|
if !headInScope && len(kept) == 0 {
|
|
return nil
|
|
}
|
|
if headInScope && len(kept) == len(resp.Events) && len(resp.LogFileRefs) == 0 {
|
|
return resp
|
|
}
|
|
scoped := &filer_pb.SubscribeMetadataResponse{
|
|
TsNs: resp.TsNs,
|
|
Events: kept,
|
|
FlushedTsNs: resp.FlushedTsNs,
|
|
}
|
|
if headInScope {
|
|
scoped.Directory = resp.Directory
|
|
scoped.EventNotification = resp.EventNotification
|
|
scoped.MutationTsNs = resp.MutationTsNs
|
|
} else {
|
|
// the batch head keeps its position as an empty marker
|
|
scoped.EventNotification = &filer_pb.EventNotification{}
|
|
}
|
|
return scoped
|
|
}
|
|
|
|
func metadataEventInScope(event *filer_pb.SubscribeMetadataResponse, prefixes []string) bool {
|
|
if event.EventNotification == nil || filer_pb.IsEmpty(event) {
|
|
return true
|
|
}
|
|
return security.MetadataPathInScope(filer_pb.MetadataEventSourceFullPath(event), prefixes) &&
|
|
security.MetadataPathInScope(filer_pb.MetadataEventTargetFullPath(event), prefixes)
|
|
}
|
|
|
|
var errMetadataSubscriptionExpired = status.Error(codes.Unauthenticated, "metadata subscription token expired")
|
|
|
|
// scopedMetadataStream ends its context when the token expires, so an open
|
|
// subscription stops there.
|
|
type scopedMetadataStream struct {
|
|
ctx context.Context
|
|
scope *security.MetadataSubscriptionScope
|
|
withheld *int64
|
|
}
|
|
|
|
// scopedClientEpoch is below every other epoch, so a scoped registration
|
|
// never replaces another one, even if its id gets taken after it was chosen.
|
|
const scopedClientEpoch = math.MinInt32
|
|
|
|
// unusedClientId gives a scoped caller a registration id of its own, so it
|
|
// cannot replace or squat on another subscriber's registration.
|
|
func (fs *FilerServer) unusedClientId() int32 {
|
|
fs.knownListenersLock.Lock()
|
|
defer fs.knownListenersLock.Unlock()
|
|
for {
|
|
id := rand.Int32N(1<<31-1) + 1
|
|
_, found := fs.knownListeners[id]
|
|
_, foundLocal := fs.knownListeners[-id]
|
|
if !found && !foundLocal {
|
|
return id
|
|
}
|
|
}
|
|
}
|
|
|
|
func newScopedMetadataStream(parent context.Context, scope *security.MetadataSubscriptionScope) (scopedMetadataStream, context.CancelFunc) {
|
|
ctx, cancel := context.WithDeadline(parent, scope.ExpiresAt)
|
|
return scopedMetadataStream{ctx: ctx, scope: scope, withheld: new(int64)}, cancel
|
|
}
|
|
|
|
func (s scopedMetadataStream) Context() context.Context { return s.ctx }
|
|
|
|
func (s scopedMetadataStream) send(resp *filer_pb.SubscribeMetadataResponse, send func(*filer_pb.SubscribeMetadataResponse) error) error {
|
|
if !time.Now().Before(s.scope.ExpiresAt) {
|
|
return errMetadataSubscriptionExpired
|
|
}
|
|
if scoped := scopeMetadataResponse(resp, s.scope.Prefixes); scoped != nil {
|
|
if err := send(scoped); err != nil {
|
|
return err
|
|
}
|
|
*s.withheld = 0
|
|
return nil
|
|
}
|
|
if resp.EventNotification == nil {
|
|
return nil
|
|
}
|
|
// withheld events still move the subscriber's resume position
|
|
*s.withheld += int64(1 + len(resp.Events))
|
|
if *s.withheld <= MaxUnsyncedEvents {
|
|
return nil
|
|
}
|
|
tsNs := resp.TsNs
|
|
for _, e := range resp.Events {
|
|
tsNs = max(tsNs, e.TsNs)
|
|
}
|
|
if err := send(&filer_pb.SubscribeMetadataResponse{EventNotification: &filer_pb.EventNotification{}, TsNs: tsNs}); err != nil {
|
|
return err
|
|
}
|
|
*s.withheld = 0
|
|
return nil
|
|
}
|
|
|
|
// sendMsg filters SendMsg too, so the embedded stream never sends unfiltered.
|
|
func (s scopedMetadataStream) sendMsg(m any, send func(*filer_pb.SubscribeMetadataResponse) error) error {
|
|
resp, ok := m.(*filer_pb.SubscribeMetadataResponse)
|
|
if !ok {
|
|
return fmt.Errorf("scoped metadata stream: unexpected message %T", m)
|
|
}
|
|
return s.send(resp, send)
|
|
}
|
|
|
|
// endErr reports an expired token in place of whatever ended the stream.
|
|
func (s scopedMetadataStream) endErr(err error) error {
|
|
if !time.Now().Before(s.scope.ExpiresAt) {
|
|
return errMetadataSubscriptionExpired
|
|
}
|
|
return err
|
|
}
|
|
|
|
type scopedSubscribeStream struct {
|
|
filer_pb.SeaweedFiler_SubscribeMetadataServer
|
|
scopedMetadataStream
|
|
}
|
|
|
|
func (s *scopedSubscribeStream) Context() context.Context { return s.ctx }
|
|
|
|
func (s *scopedSubscribeStream) Send(resp *filer_pb.SubscribeMetadataResponse) error {
|
|
return s.send(resp, s.SeaweedFiler_SubscribeMetadataServer.Send)
|
|
}
|
|
|
|
func (s *scopedSubscribeStream) SendMsg(m any) error {
|
|
return s.sendMsg(m, s.SeaweedFiler_SubscribeMetadataServer.Send)
|
|
}
|
|
|
|
type scopedSubscribeLocalStream struct {
|
|
filer_pb.SeaweedFiler_SubscribeLocalMetadataServer
|
|
scopedMetadataStream
|
|
}
|
|
|
|
func (s *scopedSubscribeLocalStream) Context() context.Context { return s.ctx }
|
|
|
|
func (s *scopedSubscribeLocalStream) Send(resp *filer_pb.SubscribeMetadataResponse) error {
|
|
return s.send(resp, s.SeaweedFiler_SubscribeLocalMetadataServer.Send)
|
|
}
|
|
|
|
func (s *scopedSubscribeLocalStream) SendMsg(m any) error {
|
|
return s.sendMsg(m, s.SeaweedFiler_SubscribeLocalMetadataServer.Send)
|
|
}
|