mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
filer: scoped read-only tokens for metadata subscriptions (#12767)
* security: scoped read-only token for filer metadata subscriptions A token signed with jwt.filer_signing.subscribe.key names the subscribe methods and the path prefixes its holder may see. mTLS peers and admin tokens stay unrestricted. Internal subscribers attach an admin token when both keys are configured. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: enforce subscription token scope on metadata streams With jwt.filer_signing.subscribe.key set, SubscribeMetadata and SubscribeLocalMetadata authorize the caller first. A scoped caller gets a stream that sends only events whose paths are all under its prefixes, filters batches, and never carries log file refs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: send an admin token on internal metadata subscriptions Peer filers, metadata followers and the lifecycle reader attach an admin token when the subscribe key is configured, so they keep the full stream without mTLS. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * shell: add fs.meta.token to mint metadata subscription tokens Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: end scoped metadata subscriptions at token expiry The scope now carries the token's exp. The scoped stream's context ends at that time and the call returns Unauthenticated, so an open stream no longer outlives its token. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer.sync: sign metadata subscriptions with the source cluster's key MetadataFollowOption takes a FilerJwtProvider; filer.sync passes the one loaded from -a.security or -b.security, so clusters with different admin keys both accept the subscription. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * shell: report fs.meta.token flag errors Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: give scoped metadata subscribers their own client id A scoped caller could reuse another subscriber's ClientId with a higher epoch and replace that registration, disconnecting it. Scoped streams now register under a server-chosen id that no other subscriber holds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * security: accept only HS256 metadata subscription tokens Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * security: test HS384 metadata subscription token rejection Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: never let a scoped subscriber replace another registration The server-chosen id can be taken by another subscriber between choosing and registering it. Registering scoped streams with the lowest epoch makes addClient refuse them rather than replace the other registration. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: send position markers for events a scoped token withholds Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test a scoped stream withholding a whole event batch Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: filter SendMsg on scoped metadata streams The scoped wrappers only overrode Send, so a SendMsg on the embedded stream would reach the client unfiltered. Route it through the same filter and refuse any other message type. Ported from #12790. Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test a scoped subscription through the real subscribe loop Run SubscribeLocalMetadata with a scoped token over a flushed backlog, the in-memory tail and live events: only in-scope events arrive, sibling, "..", encoded and cross-boundary rename paths are dropped, and no log file references are sent although the client asks for them. Without the subscribe key every event still arrives. Ported from #12790. Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test that the HTTP API refuses metadata subscription tokens GET, HEAD, PUT, POST and DELETE all refuse a subscription token, also when the subscribe key equals the read or write key. Ported from #12790. Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com>
This commit is contained in:
12 files changed
+1038
-4
No files matched your search
@@ -495,6 +495,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
|
||||
StartTsNs: sourceFilerOffsetTsNs,
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.RetryForeverOnError,
|
||||
FilerJwtProvider: sourceJwtProvider,
|
||||
GetResumeTsNs: func() int64 {
|
||||
return resumeTsNsFor(processor)
|
||||
},
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
# The JWT signing keys map to:
|
||||
# WEED_JWT_SIGNING_KEY, WEED_JWT_SIGNING_READ_KEY (master <-> volume)
|
||||
# WEED_JWT_FILER_SIGNING_KEY, WEED_JWT_FILER_SIGNING_READ_KEY (s3/clients <-> filer)
|
||||
# WEED_JWT_FILER_SIGNING_SUBSCRIBE_KEY (filer metadata subscriptions)
|
||||
# A set env var wins over the file, so the file can stay empty (or absent).
|
||||
|
||||
# comma separated origins allowed to make requests to the filer and s3 gateway.
|
||||
@@ -75,6 +76,18 @@ expires_after_seconds = 10 # seconds
|
||||
key = ""
|
||||
expires_after_seconds = 10 # seconds
|
||||
|
||||
# Optional. If this key is set, the filer's SubscribeMetadata and
|
||||
# SubscribeLocalMetadata gRPC calls require an mTLS-verified peer, an admin
|
||||
# token signed with jwt.filer_signing.key, or a read-only token signed with this
|
||||
# key. That token is an HS256 JWT with "exp", "allowed_methods" (a subset of
|
||||
# SubscribeMetadata and SubscribeLocalMetadata) and "allowed_prefixes"
|
||||
# (absolute paths); its holder only receives events whose paths are all under
|
||||
# one of the prefixes. Mint one with "fs.meta.token" in weed shell. Without
|
||||
# mTLS, set jwt.filer_signing.key and this key on every filer, S3 gateway,
|
||||
# mount and filer.sync too, so they keep full access.
|
||||
[jwt.filer_signing.subscribe]
|
||||
key = ""
|
||||
|
||||
# gRPC mTLS configuration
|
||||
# All gRPC TLS authentications are mutual (mTLS)
|
||||
# The values for ca, cert, and key are paths to the certificate/key files
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
@@ -724,7 +725,7 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
|
||||
return NewChunkStreamReaderFromLookup(ctx, lookupFn, chunks), nil
|
||||
}
|
||||
|
||||
stream, err := client.SubscribeLocalMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
|
||||
stream, err := client.SubscribeLocalMetadata(security.WithMetadataSubscriptionCredentials(ctx, nil), &filer_pb.SubscribeMetadataRequest{
|
||||
ClientName: "filer:" + string(self),
|
||||
PathPrefix: "/",
|
||||
SinceNs: lastTsNs,
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
@@ -39,6 +40,9 @@ type MetadataFollowOption struct {
|
||||
// the server sends log file chunk fids instead of streaming events,
|
||||
// and the client reads directly from volume servers.
|
||||
LogFileReaderFn LogFileReaderFn
|
||||
// FilerJwtProvider, when non-nil, signs the subscription with the source
|
||||
// cluster's own filer key instead of this process's configuration.
|
||||
FilerJwtProvider security.FilerJwtProvider
|
||||
// OnIdleHeartbeat, when non-nil, opts in to idle heartbeats: while the
|
||||
// subscriber is caught up the server periodically sends an empty response
|
||||
// carrying the current time, and this is called with that timestamp. It is
|
||||
@@ -91,7 +95,7 @@ func makeSubscribeMetadataFunc(option *MetadataFollowOption, processEventFn Proc
|
||||
if option.GetResumeTsNs != nil {
|
||||
sinceNs = option.GetResumeTsNs()
|
||||
}
|
||||
stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
|
||||
stream, err := client.SubscribeMetadata(security.WithMetadataSubscriptionCredentials(ctx, option.FilerJwtProvider), &filer_pb.SubscribeMetadataRequest{
|
||||
ClientName: option.ClientName,
|
||||
PathPrefix: option.PathPrefix,
|
||||
PathPrefixes: option.AdditionalPathPrefixes,
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
)
|
||||
|
||||
// Event is one in-shard meta-log event delivered to the router.
|
||||
@@ -157,7 +158,7 @@ func (r *Reader) Run(ctx context.Context, client filer_pb.SeaweedFilerClient, cl
|
||||
// Own context: aborting the RPC is the only way to unblock Recv.
|
||||
streamCtx, cancelStream := context.WithCancel(ctx)
|
||||
defer cancelStream()
|
||||
stream, err := client.SubscribeMetadata(streamCtx, &filer_pb.SubscribeMetadataRequest{
|
||||
stream, err := client.SubscribeMetadata(security.WithMetadataSubscriptionCredentials(streamCtx, nil), &filer_pb.SubscribeMetadataRequest{
|
||||
ClientName: clientName,
|
||||
PathPrefix: r.BucketsPath,
|
||||
SinceNs: sinceNs,
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
package security
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
jwt "github.com/golang-jwt/jwt/v5"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/metadata"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// The filer gRPC methods a scoped subscription token may name in allowed_methods.
|
||||
const (
|
||||
MethodSubscribeMetadata = "SubscribeMetadata"
|
||||
MethodSubscribeLocalMetadata = "SubscribeLocalMetadata"
|
||||
)
|
||||
|
||||
// GenJwtForMetadataSubscription mints a read-only token for the filer's
|
||||
// metadata subscription calls, limited to events under prefixes. It carries
|
||||
// allowed_prefixes and allowed_methods, so it is never an admin token.
|
||||
// Returns an empty string without a key, an expiry, or an absolute prefix.
|
||||
func GenJwtForMetadataSubscription(signingKey SigningKey, expiresAfterSec int, prefixes []string) EncodedJwt {
|
||||
prefixes, ok := cleanMetadataPrefixes(prefixes)
|
||||
if len(signingKey) == 0 || expiresAfterSec <= 0 || !ok {
|
||||
return ""
|
||||
}
|
||||
claims := SeaweedFilerClaims{
|
||||
AllowedPrefixes: prefixes,
|
||||
AllowedMethods: []string{MethodSubscribeMetadata, MethodSubscribeLocalMetadata},
|
||||
RegisteredClaims: jwt.RegisteredClaims{
|
||||
ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Second * time.Duration(expiresAfterSec))),
|
||||
},
|
||||
}
|
||||
encoded, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(signingKey))
|
||||
if err != nil {
|
||||
glog.V(0).Infof("Failed to sign metadata subscription claims: %v", err)
|
||||
return ""
|
||||
}
|
||||
return EncodedJwt(encoded)
|
||||
}
|
||||
|
||||
// cleanMetadataPrefixes cleans prefixes; ok is false when the list is empty
|
||||
// or any prefix is not absolute.
|
||||
func cleanMetadataPrefixes(prefixes []string) (cleaned []string, ok bool) {
|
||||
for _, p := range prefixes {
|
||||
if !strings.HasPrefix(p, "/") {
|
||||
return nil, false
|
||||
}
|
||||
cleaned = append(cleaned, path.Clean(p))
|
||||
}
|
||||
return cleaned, len(cleaned) > 0
|
||||
}
|
||||
|
||||
// MetadataPathInScope reports whether fullPath is one of the prefixes or below
|
||||
// one, by whole path segments: "/buckets/a" holds "/buckets/a/x", not "/buckets/ab".
|
||||
func MetadataPathInScope(fullPath string, prefixes []string) bool {
|
||||
fullPath = path.Clean("/" + fullPath)
|
||||
for _, p := range prefixes {
|
||||
if p == "/" || fullPath == p || strings.HasPrefix(fullPath, p+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// MetadataSubscriptionScope limits a subscription to events under Prefixes
|
||||
// until ExpiresAt.
|
||||
type MetadataSubscriptionScope struct {
|
||||
Prefixes []string
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
func metadataSubscriptionScope(signingKey SigningKey, token EncodedJwt, method string) (*MetadataSubscriptionScope, bool) {
|
||||
parsed, err := decodeJwt(signingKey, token, &SeaweedFilerClaims{}, jwt.WithValidMethods([]string{jwt.SigningMethodHS256.Alg()}))
|
||||
if err != nil || parsed == nil || !parsed.Valid {
|
||||
return nil, false
|
||||
}
|
||||
claims, ok := parsed.Claims.(*SeaweedFilerClaims)
|
||||
if !ok || claims.SessionId != "" || claims.ExpiresAt == nil {
|
||||
return nil, false
|
||||
}
|
||||
allowsMethod := false
|
||||
for _, m := range claims.AllowedMethods {
|
||||
if m != MethodSubscribeMetadata && m != MethodSubscribeLocalMetadata {
|
||||
return nil, false
|
||||
}
|
||||
allowsMethod = allowsMethod || m == method
|
||||
}
|
||||
if !allowsMethod {
|
||||
return nil, false
|
||||
}
|
||||
prefixes, ok := cleanMetadataPrefixes(claims.AllowedPrefixes)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
return &MetadataSubscriptionScope{Prefixes: prefixes, ExpiresAt: claims.ExpiresAt.Time}, true
|
||||
}
|
||||
|
||||
// CheckGrpcMetadataSubscription authorizes a metadata subscription call.
|
||||
// Without subscribeKey every caller is unrestricted (nil scope). With it, an
|
||||
// mTLS-verified peer or an admin token signed by adminKey is unrestricted,
|
||||
// and a token signed by subscribeKey is limited to the returned scope.
|
||||
func CheckGrpcMetadataSubscription(ctx context.Context, subscribeKey, adminKey SigningKey, method string) (scope *MetadataSubscriptionScope, err error) {
|
||||
if len(subscribeKey) == 0 || isMtlsVerifiedPeer(ctx) {
|
||||
return nil, nil
|
||||
}
|
||||
md, _ := metadata.FromIncomingContext(ctx)
|
||||
authHeaders := md.Get("authorization")
|
||||
if len(authHeaders) > maxGrpcAdminAuthValues {
|
||||
authHeaders = authHeaders[:maxGrpcAdminAuthValues]
|
||||
}
|
||||
for _, raw := range authHeaders {
|
||||
parts := strings.Fields(raw)
|
||||
if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") || parts[1] == "" {
|
||||
continue
|
||||
}
|
||||
token := EncodedJwt(parts[1])
|
||||
if IsFilerAdminToken(adminKey, token) {
|
||||
return nil, nil
|
||||
}
|
||||
if scope, ok := metadataSubscriptionScope(subscribeKey, token, method); ok {
|
||||
return scope, nil
|
||||
}
|
||||
}
|
||||
return nil, status.Error(codes.Unauthenticated, "metadata subscription requires mTLS, an admin token, or a subscription token for this method")
|
||||
}
|
||||
|
||||
// WithMetadataSubscriptionCredentials adds an admin token to a subscription
|
||||
// call: from jwtProvider when the caller has per-cluster keys, else when this
|
||||
// process has both jwt.filer_signing.key and jwt.filer_signing.subscribe.key,
|
||||
// so internal subscribers keep full access.
|
||||
func WithMetadataSubscriptionCredentials(ctx context.Context, jwtProvider FilerJwtProvider) context.Context {
|
||||
var token EncodedJwt
|
||||
if jwtProvider != nil {
|
||||
token = jwtProvider(true)
|
||||
} else if v := util.GetViper(); v.GetString("jwt.filer_signing.subscribe.key") != "" {
|
||||
token = GenJwtForFilerAdmin(SigningKey(v.GetString("jwt.filer_signing.key")), v.GetInt("jwt.filer_signing.expires_after_seconds"))
|
||||
}
|
||||
if token == "" {
|
||||
return ctx
|
||||
}
|
||||
return metadata.AppendToOutgoingContext(ctx, "authorization", BearerPrefix+string(token))
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package security
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
jwt "github.com/golang-jwt/jwt/v5"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/metadata"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
func incomingBearer(token string) context.Context {
|
||||
return metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer "+token))
|
||||
}
|
||||
|
||||
func TestCheckGrpcMetadataSubscription(t *testing.T) {
|
||||
subKey := SigningKey("subscribe-key")
|
||||
adminKey := SigningKey("admin-key")
|
||||
scoped := string(GenJwtForMetadataSubscription(subKey, 60, []string{"/buckets/uploads", "/buckets/logs/"}))
|
||||
if scoped == "" {
|
||||
t.Fatal("no scoped token")
|
||||
}
|
||||
signWith := func(method jwt.SigningMethod, key SigningKey, claims jwt.Claims) string {
|
||||
s, err := jwt.NewWithClaims(method, claims).SignedString([]byte(key))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
sign := func(key SigningKey, claims jwt.Claims) string { return signWith(jwt.SigningMethodHS256, key, claims) }
|
||||
exp := jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Minute))}
|
||||
subscribeOnly := []string{MethodSubscribeMetadata}
|
||||
wantScope := []string{"/buckets/uploads", "/buckets/logs"}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
ctx context.Context
|
||||
subKey SigningKey
|
||||
method string
|
||||
wantScope []string
|
||||
wantCode codes.Code
|
||||
}{
|
||||
{"feature off keeps the call open", context.Background(), nil, MethodSubscribeMetadata, nil, codes.OK},
|
||||
{"scoped token", incomingBearer(scoped), subKey, MethodSubscribeMetadata, wantScope, codes.OK},
|
||||
{"scoped token, local call", incomingBearer(scoped), subKey, MethodSubscribeLocalMetadata, wantScope, codes.OK},
|
||||
{"admin token is unrestricted", incomingBearer(string(GenJwtForFilerAdmin(adminKey, 60))), subKey, MethodSubscribeMetadata, nil, codes.OK},
|
||||
{"no token", context.Background(), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated},
|
||||
{"scoped token signed with another key", incomingBearer(string(GenJwtForMetadataSubscription(SigningKey("other"), 60, []string{"/a"}))), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated},
|
||||
{"scoped token signed with the admin key", incomingBearer(string(GenJwtForMetadataSubscription(adminKey, 60, []string{"/a"}))), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated},
|
||||
{"admin token signed with the subscribe key", incomingBearer(string(GenJwtForFilerAdmin(subKey, 60))), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated},
|
||||
{"token without prefixes", incomingBearer(sign(subKey, SeaweedFilerClaims{AllowedMethods: subscribeOnly, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated},
|
||||
{"token without methods", incomingBearer(sign(subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a"}, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated},
|
||||
{"token for a write method too", incomingBearer(sign(subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a"}, AllowedMethods: []string{MethodSubscribeMetadata, "PUT"}, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated},
|
||||
{"token for the other method only", incomingBearer(sign(subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a"}, AllowedMethods: []string{MethodSubscribeLocalMetadata}, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated},
|
||||
{"relative prefix", incomingBearer(sign(subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a", "b"}, AllowedMethods: subscribeOnly, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated},
|
||||
{"session token", incomingBearer(sign(subKey, SeaweedFilerClaims{SessionId: "s", AllowedPrefixes: []string{"/a"}, AllowedMethods: subscribeOnly, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated},
|
||||
{"token without expiry", incomingBearer(sign(subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a"}, AllowedMethods: subscribeOnly})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated},
|
||||
{"HS384 token", incomingBearer(signWith(jwt.SigningMethodHS384, subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a"}, AllowedMethods: subscribeOnly, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated},
|
||||
{"HS512 token", incomingBearer(signWith(jwt.SigningMethodHS512, subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a"}, AllowedMethods: subscribeOnly, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated},
|
||||
{"expired token", incomingBearer(sign(subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a"}, AllowedMethods: subscribeOnly, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(time.Now().Add(-time.Hour))}})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
scope, err := CheckGrpcMetadataSubscription(tt.ctx, tt.subKey, adminKey, tt.method)
|
||||
if status.Code(err) != tt.wantCode {
|
||||
t.Fatalf("code = %v (%v), want %v", status.Code(err), err, tt.wantCode)
|
||||
}
|
||||
if (scope == nil) != (tt.wantScope == nil) || scope != nil && !slices.Equal(scope.Prefixes, tt.wantScope) {
|
||||
t.Fatalf("scope = %+v, want %v", scope, tt.wantScope)
|
||||
}
|
||||
if scope != nil && time.Until(scope.ExpiresAt) <= 0 {
|
||||
t.Fatalf("scope expiry %v is not in the future", scope.ExpiresAt)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenJwtForMetadataSubscriptionRefusesUnusableScope(t *testing.T) {
|
||||
key := SigningKey("k")
|
||||
for name, tok := range map[string]EncodedJwt{
|
||||
"no expiry": GenJwtForMetadataSubscription(key, 0, []string{"/a"}),
|
||||
"no prefix": GenJwtForMetadataSubscription(key, 60, nil),
|
||||
"relative prefix": GenJwtForMetadataSubscription(key, 60, []string{"/a", "b"}),
|
||||
"no key": GenJwtForMetadataSubscription(nil, 60, []string{"/a"}),
|
||||
} {
|
||||
if tok != "" {
|
||||
t.Errorf("%s: minted %q", name, tok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopedSubscriptionTokenIsNotAdmin(t *testing.T) {
|
||||
key := SigningKey("same-key")
|
||||
if IsFilerAdminToken(key, GenJwtForMetadataSubscription(key, 60, []string{"/a"})) {
|
||||
t.Fatal("a scoped subscription token must not count as an admin token")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataPathInScope(t *testing.T) {
|
||||
prefixes := []string{"/buckets/a"}
|
||||
for p, want := range map[string]bool{
|
||||
"/buckets/a": true,
|
||||
"/buckets/a/x/y": true,
|
||||
"/buckets/ab": false,
|
||||
"/buckets": false,
|
||||
"/buckets/a/../b/x": false,
|
||||
"/buckets/a/../a/x": true,
|
||||
"/": false,
|
||||
} {
|
||||
if got := MetadataPathInScope(p, prefixes); got != want {
|
||||
t.Errorf("MetadataPathInScope(%q) = %v, want %v", p, got, want)
|
||||
}
|
||||
}
|
||||
if !MetadataPathInScope("/anything", []string{"/"}) {
|
||||
t.Error("the root prefix holds every path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithMetadataSubscriptionCredentials(t *testing.T) {
|
||||
v := util.GetViper()
|
||||
defer v.Set("jwt.filer_signing.key", "")
|
||||
defer v.Set("jwt.filer_signing.subscribe.key", "")
|
||||
|
||||
v.Set("jwt.filer_signing.key", "admin-key")
|
||||
if md, _ := metadata.FromOutgoingContext(WithMetadataSubscriptionCredentials(context.Background(), nil)); len(md.Get("authorization")) != 0 {
|
||||
t.Fatal("no admin token without the subscribe key")
|
||||
}
|
||||
|
||||
v.Set("jwt.filer_signing.subscribe.key", "subscribe-key")
|
||||
md, _ := metadata.FromOutgoingContext(WithMetadataSubscriptionCredentials(context.Background(), nil))
|
||||
in := metadata.NewIncomingContext(context.Background(), md)
|
||||
scope, err := CheckGrpcMetadataSubscription(in, SigningKey("subscribe-key"), SigningKey("admin-key"), MethodSubscribeLocalMetadata)
|
||||
if err != nil || scope != nil {
|
||||
t.Fatalf("scope=%v err=%v, want unrestricted", scope, err)
|
||||
}
|
||||
|
||||
clusterKey := SigningKey("cluster-b-key")
|
||||
md, _ = metadata.FromOutgoingContext(WithMetadataSubscriptionCredentials(context.Background(), func(bool) EncodedJwt {
|
||||
return GenJwtForFilerServer(clusterKey, 10)
|
||||
}))
|
||||
if len(md.Get("authorization")) != 1 {
|
||||
t.Fatalf("want only the per-cluster token, got %v", md.Get("authorization"))
|
||||
}
|
||||
scope, err = CheckGrpcMetadataSubscription(metadata.NewIncomingContext(context.Background(), md), SigningKey("subscribe-b"), clusterKey, MethodSubscribeMetadata)
|
||||
if err != nil || scope != nil {
|
||||
t.Fatalf("per-cluster token: scope=%v err=%v, want unrestricted", scope, err)
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
|
||||
)
|
||||
@@ -640,6 +641,18 @@ func (p *gapPass) park(ctx context.Context, cursor *log_buffer.MessagePosition,
|
||||
}
|
||||
|
||||
func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, stream filer_pb.SeaweedFiler_SubscribeMetadataServer) (err error) {
|
||||
scope, err := metadataSubscriptionScope(stream.Context(), security.MethodSubscribeMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if scope != nil {
|
||||
req.ClientSupportsMetadataChunks = false
|
||||
req.ClientId, req.ClientEpoch = fs.unusedClientId(), scopedClientEpoch
|
||||
scoped, cancel := newScopedMetadataStream(stream.Context(), scope)
|
||||
defer cancel()
|
||||
defer func() { err = scoped.endErr(err) }()
|
||||
stream = &scopedSubscribeStream{SeaweedFiler_SubscribeMetadataServer: stream, scopedMetadataStream: scoped}
|
||||
}
|
||||
// A filer that has not learned remote peers yet serves the local log and
|
||||
// upgrades when the first one appears. RemotePeerArrivedChan takes the
|
||||
// arrival channel under the same lock as the peer check, so a peer
|
||||
@@ -1122,7 +1135,19 @@ func (fs *FilerServer) endOfSubscription(stream context.Context, err error) erro
|
||||
return err
|
||||
}
|
||||
|
||||
func (fs *FilerServer) SubscribeLocalMetadata(req *filer_pb.SubscribeMetadataRequest, stream filer_pb.SeaweedFiler_SubscribeLocalMetadataServer) error {
|
||||
func (fs *FilerServer) SubscribeLocalMetadata(req *filer_pb.SubscribeMetadataRequest, stream filer_pb.SeaweedFiler_SubscribeLocalMetadataServer) (err error) {
|
||||
scope, err := metadataSubscriptionScope(stream.Context(), security.MethodSubscribeLocalMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if scope != nil {
|
||||
req.ClientSupportsMetadataChunks = false
|
||||
req.ClientId, req.ClientEpoch = fs.unusedClientId(), scopedClientEpoch
|
||||
scoped, cancel := newScopedMetadataStream(stream.Context(), scope)
|
||||
defer cancel()
|
||||
defer func() { err = scoped.endErr(err) }()
|
||||
stream = &scopedSubscribeLocalStream{SeaweedFiler_SubscribeLocalMetadataServer: stream, scopedMetadataStream: scoped}
|
||||
}
|
||||
return fs.subscribeLocalMetadata(req, stream, nil)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"math/rand/v2"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
func metadataSubscriptionScope(ctx context.Context, method string) (*security.MetadataSubscriptionScope, error) {
|
||||
v := util.GetViper()
|
||||
return security.CheckGrpcMetadataSubscription(ctx,
|
||||
security.SigningKey(v.GetString("jwt.filer_signing.subscribe.key")),
|
||||
security.SigningKey(v.GetString("jwt.filer_signing.key")),
|
||||
method)
|
||||
}
|
||||
|
||||
// scopeMetadataResponse keeps only what a scoped subscriber may see: events
|
||||
// whose every path is in scope, so a rename across the boundary never shows
|
||||
// the far side, and no log file refs. Returns nil when nothing is left.
|
||||
func scopeMetadataResponse(resp *filer_pb.SubscribeMetadataResponse, prefixes []string) *filer_pb.SubscribeMetadataResponse {
|
||||
if resp.EventNotification == nil && len(resp.LogFileRefs) > 0 {
|
||||
return nil
|
||||
}
|
||||
var kept []*filer_pb.SubscribeMetadataResponse
|
||||
for _, e := range resp.Events {
|
||||
if metadataEventInScope(e, prefixes) {
|
||||
kept = append(kept, e)
|
||||
}
|
||||
}
|
||||
headInScope := metadataEventInScope(resp, prefixes)
|
||||
if !headInScope && len(kept) == 0 {
|
||||
return nil
|
||||
}
|
||||
if headInScope && len(kept) == len(resp.Events) && len(resp.LogFileRefs) == 0 {
|
||||
return resp
|
||||
}
|
||||
scoped := &filer_pb.SubscribeMetadataResponse{
|
||||
TsNs: resp.TsNs,
|
||||
Events: kept,
|
||||
FlushedTsNs: resp.FlushedTsNs,
|
||||
}
|
||||
if headInScope {
|
||||
scoped.Directory = resp.Directory
|
||||
scoped.EventNotification = resp.EventNotification
|
||||
scoped.MutationTsNs = resp.MutationTsNs
|
||||
} else {
|
||||
// the batch head keeps its position as an empty marker
|
||||
scoped.EventNotification = &filer_pb.EventNotification{}
|
||||
}
|
||||
return scoped
|
||||
}
|
||||
|
||||
func metadataEventInScope(event *filer_pb.SubscribeMetadataResponse, prefixes []string) bool {
|
||||
if event.EventNotification == nil || filer_pb.IsEmpty(event) {
|
||||
return true
|
||||
}
|
||||
return security.MetadataPathInScope(filer_pb.MetadataEventSourceFullPath(event), prefixes) &&
|
||||
security.MetadataPathInScope(filer_pb.MetadataEventTargetFullPath(event), prefixes)
|
||||
}
|
||||
|
||||
var errMetadataSubscriptionExpired = status.Error(codes.Unauthenticated, "metadata subscription token expired")
|
||||
|
||||
// scopedMetadataStream ends its context when the token expires, so an open
|
||||
// subscription stops there.
|
||||
type scopedMetadataStream struct {
|
||||
ctx context.Context
|
||||
scope *security.MetadataSubscriptionScope
|
||||
withheld *int64
|
||||
}
|
||||
|
||||
// scopedClientEpoch is below every other epoch, so a scoped registration
|
||||
// never replaces another one, even if its id gets taken after it was chosen.
|
||||
const scopedClientEpoch = math.MinInt32
|
||||
|
||||
// unusedClientId gives a scoped caller a registration id of its own, so it
|
||||
// cannot replace or squat on another subscriber's registration.
|
||||
func (fs *FilerServer) unusedClientId() int32 {
|
||||
fs.knownListenersLock.Lock()
|
||||
defer fs.knownListenersLock.Unlock()
|
||||
for {
|
||||
id := rand.Int32N(1<<31-1) + 1
|
||||
_, found := fs.knownListeners[id]
|
||||
_, foundLocal := fs.knownListeners[-id]
|
||||
if !found && !foundLocal {
|
||||
return id
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newScopedMetadataStream(parent context.Context, scope *security.MetadataSubscriptionScope) (scopedMetadataStream, context.CancelFunc) {
|
||||
ctx, cancel := context.WithDeadline(parent, scope.ExpiresAt)
|
||||
return scopedMetadataStream{ctx: ctx, scope: scope, withheld: new(int64)}, cancel
|
||||
}
|
||||
|
||||
func (s scopedMetadataStream) Context() context.Context { return s.ctx }
|
||||
|
||||
func (s scopedMetadataStream) send(resp *filer_pb.SubscribeMetadataResponse, send func(*filer_pb.SubscribeMetadataResponse) error) error {
|
||||
if !time.Now().Before(s.scope.ExpiresAt) {
|
||||
return errMetadataSubscriptionExpired
|
||||
}
|
||||
if scoped := scopeMetadataResponse(resp, s.scope.Prefixes); scoped != nil {
|
||||
if err := send(scoped); err != nil {
|
||||
return err
|
||||
}
|
||||
*s.withheld = 0
|
||||
return nil
|
||||
}
|
||||
if resp.EventNotification == nil {
|
||||
return nil
|
||||
}
|
||||
// withheld events still move the subscriber's resume position
|
||||
*s.withheld += int64(1 + len(resp.Events))
|
||||
if *s.withheld <= MaxUnsyncedEvents {
|
||||
return nil
|
||||
}
|
||||
tsNs := resp.TsNs
|
||||
for _, e := range resp.Events {
|
||||
tsNs = max(tsNs, e.TsNs)
|
||||
}
|
||||
if err := send(&filer_pb.SubscribeMetadataResponse{EventNotification: &filer_pb.EventNotification{}, TsNs: tsNs}); err != nil {
|
||||
return err
|
||||
}
|
||||
*s.withheld = 0
|
||||
return nil
|
||||
}
|
||||
|
||||
// sendMsg filters SendMsg too, so the embedded stream never sends unfiltered.
|
||||
func (s scopedMetadataStream) sendMsg(m any, send func(*filer_pb.SubscribeMetadataResponse) error) error {
|
||||
resp, ok := m.(*filer_pb.SubscribeMetadataResponse)
|
||||
if !ok {
|
||||
return fmt.Errorf("scoped metadata stream: unexpected message %T", m)
|
||||
}
|
||||
return s.send(resp, send)
|
||||
}
|
||||
|
||||
// endErr reports an expired token in place of whatever ended the stream.
|
||||
func (s scopedMetadataStream) endErr(err error) error {
|
||||
if !time.Now().Before(s.scope.ExpiresAt) {
|
||||
return errMetadataSubscriptionExpired
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type scopedSubscribeStream struct {
|
||||
filer_pb.SeaweedFiler_SubscribeMetadataServer
|
||||
scopedMetadataStream
|
||||
}
|
||||
|
||||
func (s *scopedSubscribeStream) Context() context.Context { return s.ctx }
|
||||
|
||||
func (s *scopedSubscribeStream) Send(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
return s.send(resp, s.SeaweedFiler_SubscribeMetadataServer.Send)
|
||||
}
|
||||
|
||||
func (s *scopedSubscribeStream) SendMsg(m any) error {
|
||||
return s.sendMsg(m, s.SeaweedFiler_SubscribeMetadataServer.Send)
|
||||
}
|
||||
|
||||
type scopedSubscribeLocalStream struct {
|
||||
filer_pb.SeaweedFiler_SubscribeLocalMetadataServer
|
||||
scopedMetadataStream
|
||||
}
|
||||
|
||||
func (s *scopedSubscribeLocalStream) Context() context.Context { return s.ctx }
|
||||
|
||||
func (s *scopedSubscribeLocalStream) Send(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
return s.send(resp, s.SeaweedFiler_SubscribeLocalMetadataServer.Send)
|
||||
}
|
||||
|
||||
func (s *scopedSubscribeLocalStream) SendMsg(m any) error {
|
||||
return s.sendMsg(m, s.SeaweedFiler_SubscribeLocalMetadataServer.Send)
|
||||
}
|
||||
@@ -0,0 +1,395 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/metadata"
|
||||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
func scopeTestEvent(dir, name string, tsNs int64) *filer_pb.SubscribeMetadataResponse {
|
||||
return &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: dir,
|
||||
TsNs: tsNs,
|
||||
EventNotification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: name}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopeMetadataResponse(t *testing.T) {
|
||||
prefixes := []string{"/buckets/uploads"}
|
||||
|
||||
if got := scopeMetadataResponse(scopeTestEvent("/buckets/uploads/2026", "a.txt", 1), prefixes); got == nil {
|
||||
t.Error("an event under the prefix must pass")
|
||||
}
|
||||
if got := scopeMetadataResponse(scopeTestEvent("/buckets", "uploads", 1), prefixes); got == nil {
|
||||
t.Error("an event on the prefix itself must pass")
|
||||
}
|
||||
for _, dir := range []string{"/buckets/other", "/buckets/uploads-private", "/buckets/uploads/../other"} {
|
||||
if got := scopeMetadataResponse(scopeTestEvent(dir, "a.txt", 1), prefixes); got != nil {
|
||||
t.Errorf("an event in %s must be dropped", dir)
|
||||
}
|
||||
}
|
||||
|
||||
out := scopeTestEvent("/buckets/uploads", "a.txt", 1)
|
||||
out.EventNotification.OldEntry = &filer_pb.Entry{Name: "a.txt"}
|
||||
out.EventNotification.NewParentPath = "/buckets/other"
|
||||
if got := scopeMetadataResponse(out, prefixes); got != nil {
|
||||
t.Error("a rename out of the scope must be dropped")
|
||||
}
|
||||
in := scopeTestEvent("/buckets/other", "a.txt", 1)
|
||||
in.EventNotification.OldEntry = &filer_pb.Entry{Name: "a.txt"}
|
||||
in.EventNotification.NewParentPath = "/buckets/uploads"
|
||||
if got := scopeMetadataResponse(in, prefixes); got != nil {
|
||||
t.Error("a rename into the scope must be dropped")
|
||||
}
|
||||
del := &filer_pb.SubscribeMetadataResponse{Directory: "/etc", EventNotification: &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: "x"}}}
|
||||
if got := scopeMetadataResponse(del, prefixes); got != nil {
|
||||
t.Error("a delete outside the scope must be dropped")
|
||||
}
|
||||
|
||||
for name, marker := range map[string]*filer_pb.SubscribeMetadataResponse{
|
||||
"heartbeat": {TsNs: 5},
|
||||
"flush report": {FlushedTsNs: 5},
|
||||
"empty marker": {TsNs: 5, EventNotification: &filer_pb.EventNotification{}},
|
||||
} {
|
||||
if got := scopeMetadataResponse(marker, prefixes); got != marker {
|
||||
t.Errorf("%s must pass unchanged", name)
|
||||
}
|
||||
}
|
||||
if got := scopeMetadataResponse(&filer_pb.SubscribeMetadataResponse{LogFileRefs: []*filer_pb.LogFileChunkRef{{}}}, prefixes); got != nil {
|
||||
t.Error("log file refs must never be sent")
|
||||
}
|
||||
|
||||
batch := scopeTestEvent("/buckets/other", "x", 10)
|
||||
batch.MutationTsNs = 9
|
||||
batch.FlushedTsNs = 7
|
||||
batch.Events = []*filer_pb.SubscribeMetadataResponse{scopeTestEvent("/buckets/uploads", "ok", 11), scopeTestEvent("/etc", "secret", 12)}
|
||||
got := scopeMetadataResponse(batch, prefixes)
|
||||
if got == nil || len(got.Events) != 1 || got.Events[0].Directory != "/buckets/uploads" {
|
||||
t.Fatalf("batch not filtered: %+v", got)
|
||||
}
|
||||
if got.Directory != "" || !filer_pb.IsEmpty(got) || got.MutationTsNs != 0 || got.TsNs != 10 || got.FlushedTsNs != 7 {
|
||||
t.Errorf("an out-of-scope batch head must become a marker at its position: %+v", got)
|
||||
}
|
||||
if len(batch.Events) != 2 {
|
||||
t.Error("the original response must not be modified")
|
||||
}
|
||||
|
||||
inScopeHead := scopeTestEvent("/buckets/uploads", "head", 10)
|
||||
inScopeHead.Events = []*filer_pb.SubscribeMetadataResponse{scopeTestEvent("/etc", "secret", 11)}
|
||||
got = scopeMetadataResponse(inScopeHead, prefixes)
|
||||
if got == nil || got.EventNotification != inScopeHead.EventNotification || len(got.Events) != 0 {
|
||||
t.Errorf("an in-scope head keeps its event and drops the out-of-scope tail: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopedSubscriptionEndsAtTokenExpiry(t *testing.T) {
|
||||
h := newSubscribeHarness(t)
|
||||
v := util.GetViper()
|
||||
v.Set("jwt.filer_signing.subscribe.key", "sub-key")
|
||||
t.Cleanup(func() { v.Set("jwt.filer_signing.subscribe.key", "") })
|
||||
token := security.GenJwtForMetadataSubscription(security.SigningKey("sub-key"), 2, []string{"/t"})
|
||||
ctx, cancel := context.WithCancel(metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer "+string(token))))
|
||||
defer cancel()
|
||||
stream := &fakeSubscribeStream{ctx: ctx}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- h.fs.SubscribeLocalMetadata(&filer_pb.SubscribeMetadataRequest{ClientName: "scoped", ClientId: 9, ClientEpoch: 1, SinceNs: 1}, stream)
|
||||
}()
|
||||
|
||||
now := time.Now().UnixNano()
|
||||
h.append(now)
|
||||
outside, _ := proto.Marshal(&filer_pb.SubscribeMetadataResponse{Directory: "/other", TsNs: now + 1, EventNotification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "x"}}})
|
||||
if err := h.f.LocalMetaLogBuffer.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: now + 1, Data: outside, Key: []byte("/other/x")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
if status.Code(err) != codes.Unauthenticated {
|
||||
t.Fatalf("stream ended with %v, want Unauthenticated at token expiry", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("scoped subscription outlived its token")
|
||||
}
|
||||
for _, m := range stream.snapshot() {
|
||||
if m.Directory == "/other" {
|
||||
t.Fatalf("out-of-scope event delivered: %+v", m)
|
||||
}
|
||||
}
|
||||
if got := eventTimestamps(stream.snapshot()); len(got) != 1 || got[0] != now {
|
||||
t.Fatalf("delivered %v, want only the in-scope event", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopedSubscriptionKeepsOtherRegistrations(t *testing.T) {
|
||||
h := newSubscribeHarness(t)
|
||||
v := util.GetViper()
|
||||
v.Set("jwt.filer_signing.subscribe.key", "sub-key")
|
||||
t.Cleanup(func() { v.Set("jwt.filer_signing.subscribe.key", "") })
|
||||
if replacing, _, _ := h.fs.addClient("local", "mount", "10.0.0.1:1", "/", -9, 1); !replacing {
|
||||
t.Fatal("unrestricted registration was not added")
|
||||
}
|
||||
|
||||
token := security.GenJwtForMetadataSubscription(security.SigningKey("sub-key"), 60, []string{"/t"})
|
||||
ctx, cancel := context.WithCancel(metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer "+string(token))))
|
||||
stream := &fakeSubscribeStream{ctx: ctx}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- h.fs.SubscribeLocalMetadata(&filer_pb.SubscribeMetadataRequest{ClientName: "scoped", ClientId: 9, ClientEpoch: 100, SinceNs: 1}, stream)
|
||||
}()
|
||||
|
||||
now := time.Now().UnixNano()
|
||||
h.append(now)
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for len(stream.snapshot()) == 0 {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("scoped subscription delivered nothing")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if !h.fs.hasClient(-9, 1) {
|
||||
t.Fatal("scoped caller replaced the unrestricted registration")
|
||||
}
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("scoped subscription did not end")
|
||||
}
|
||||
if !h.fs.hasClient(-9, 1) {
|
||||
t.Fatal("scoped caller removed the unrestricted registration on exit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopedRegistrationNeverReplacesLaterRegistration(t *testing.T) {
|
||||
h := newSubscribeHarness(t)
|
||||
id := h.fs.unusedClientId()
|
||||
if replacing, _, _ := h.fs.addClient("", "mount", "10.0.0.1:1", "/", id, 1); !replacing {
|
||||
t.Fatal("unrestricted registration was not added")
|
||||
}
|
||||
if replacing, alreadyKnown, _ := h.fs.addClient("", "scoped", "10.0.0.2:1", "/t", id, scopedClientEpoch); replacing || !alreadyKnown {
|
||||
t.Fatalf("scoped registration replacing=%v alreadyKnown=%v, want refused", replacing, alreadyKnown)
|
||||
}
|
||||
h.fs.deleteClient("", "scoped", id, scopedClientEpoch)
|
||||
if !h.fs.hasClient(id, 1) {
|
||||
t.Fatal("unrestricted registration lost to a scoped caller")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopedStreamMarksWithheldProgress(t *testing.T) {
|
||||
var sent []*filer_pb.SubscribeMetadataResponse
|
||||
send := func(m *filer_pb.SubscribeMetadataResponse) error {
|
||||
sent = append(sent, m)
|
||||
return nil
|
||||
}
|
||||
s, cancel := newScopedMetadataStream(context.Background(), &security.MetadataSubscriptionScope{Prefixes: []string{"/a"}, ExpiresAt: time.Now().Add(time.Hour)})
|
||||
defer cancel()
|
||||
var last int64
|
||||
for i := int64(1); i <= MaxUnsyncedEvents+1; i++ {
|
||||
last = i
|
||||
if err := s.send(scopeTestEvent("/other", "x", i), send); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(sent) != 1 {
|
||||
t.Fatalf("sent %d messages, want one position marker", len(sent))
|
||||
}
|
||||
m := sent[0]
|
||||
if m.TsNs != last || !filer_pb.IsEmpty(m) || m.Directory != "" || len(m.Events) > 0 {
|
||||
t.Fatalf("marker %+v, want an empty event at %d", m, last)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopedStreamMarksWithheldBatch(t *testing.T) {
|
||||
var sent []*filer_pb.SubscribeMetadataResponse
|
||||
send := func(m *filer_pb.SubscribeMetadataResponse) error {
|
||||
sent = append(sent, m)
|
||||
return nil
|
||||
}
|
||||
s, cancel := newScopedMetadataStream(context.Background(), &security.MetadataSubscriptionScope{Prefixes: []string{"/a"}, ExpiresAt: time.Now().Add(time.Hour)})
|
||||
defer cancel()
|
||||
batch := scopeTestEvent("/other", "head", 1)
|
||||
for i := int64(2); i <= MaxUnsyncedEvents+1; i++ {
|
||||
batch.Events = append(batch.Events, scopeTestEvent("/other", "x", i))
|
||||
}
|
||||
if err := s.send(batch, send); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(sent) != 1 {
|
||||
t.Fatalf("sent %d messages, want one position marker", len(sent))
|
||||
}
|
||||
m, want := sent[0], int64(MaxUnsyncedEvents+1)
|
||||
if m.TsNs != want || !filer_pb.IsEmpty(m) || m.Directory != "" || len(m.Events) > 0 {
|
||||
t.Fatalf("marker %+v, want an empty event at %d", m, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopedStreamSendMsgIsFiltered(t *testing.T) {
|
||||
scoped, cancel := newScopedMetadataStream(context.Background(), &security.MetadataSubscriptionScope{Prefixes: []string{"/a"}, ExpiresAt: time.Now().Add(time.Hour)})
|
||||
defer cancel()
|
||||
for _, s := range []interface{ SendMsg(any) error }{
|
||||
&scopedSubscribeStream{SeaweedFiler_SubscribeMetadataServer: &fakeSubscribeStream{ctx: context.Background()}, scopedMetadataStream: scoped},
|
||||
&scopedSubscribeLocalStream{SeaweedFiler_SubscribeLocalMetadataServer: &fakeSubscribeStream{ctx: context.Background()}, scopedMetadataStream: scoped},
|
||||
} {
|
||||
if err := s.SendMsg(scopeTestEvent("/b", "x", 1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.SendMsg(scopeTestEvent("/a", "x", 2)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.SendMsg(&filer_pb.Entry{}); err == nil {
|
||||
t.Error("an unexpected message type must be refused")
|
||||
}
|
||||
var inner *fakeSubscribeStream
|
||||
switch w := s.(type) {
|
||||
case *scopedSubscribeStream:
|
||||
inner = w.SeaweedFiler_SubscribeMetadataServer.(*fakeSubscribeStream)
|
||||
case *scopedSubscribeLocalStream:
|
||||
inner = w.SeaweedFiler_SubscribeLocalMetadataServer.(*fakeSubscribeStream)
|
||||
}
|
||||
if msgs := inner.snapshot(); len(msgs) != 1 || msgs[0].Directory != "/a" {
|
||||
t.Errorf("%T sent %v", s, msgs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func scopedLogEntry(tsNs int64, dir, name, newParent string, oldEntry bool) *filer_pb.LogEntry {
|
||||
n := &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: name}, NewParentPath: newParent}
|
||||
if oldEntry {
|
||||
n.OldEntry = &filer_pb.Entry{Name: name}
|
||||
}
|
||||
data, err := proto.Marshal(&filer_pb.SubscribeMetadataResponse{Directory: dir, EventNotification: n, TsNs: tsNs})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &filer_pb.LogEntry{TsNs: tsNs, Data: data, Key: []byte(dir + "/" + name)}
|
||||
}
|
||||
|
||||
// The real subscribe loop with a scoped token: flushed backlog, the memory
|
||||
// tail and live events are all filtered, and log file references are not sent
|
||||
// although the client asks for them.
|
||||
func TestSubscribeLocalMetadataScopedEndToEnd(t *testing.T) {
|
||||
for _, enforce := range []bool{true, false} {
|
||||
t.Run(map[bool]string{true: "scoped", false: "key unset"}[enforce], func(t *testing.T) {
|
||||
h := newSubscribeHarness(t)
|
||||
if enforce {
|
||||
v := util.GetViper()
|
||||
v.Set("jwt.filer_signing.subscribe.key", "sub-key")
|
||||
t.Cleanup(func() { v.Set("jwt.filer_signing.subscribe.key", "") })
|
||||
}
|
||||
|
||||
ts := func(i int) int64 { return h.base + int64(i)*int64(time.Millisecond) }
|
||||
entries := []*filer_pb.LogEntry{
|
||||
scopedLogEntry(ts(0), "/buckets/uploads", "in-flushed", "", false),
|
||||
scopedLogEntry(ts(1), "/buckets/uploads-private", "sibling", "", false),
|
||||
scopedLogEntry(ts(2), "/etc/iam", "identity.json", "", false),
|
||||
scopedLogEntry(ts(3), "/buckets/uploads/..", "climb", "", false),
|
||||
scopedLogEntry(ts(4), "/buckets/uploads", "renamed-out", "/buckets/other", true),
|
||||
scopedLogEntry(ts(5), "/buckets/uploads/2026", "in-memory", "", false),
|
||||
scopedLogEntry(ts(6), "/buckets/uploads%2F..%2Fother", "encoded", "", false),
|
||||
}
|
||||
for i, e := range entries {
|
||||
if err := h.f.LocalMetaLogBuffer.AddLogEntryToBuffer(e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if i == 4 {
|
||||
h.f.LocalMetaLogBuffer.ForceFlush()
|
||||
}
|
||||
}
|
||||
|
||||
token := security.GenJwtForMetadataSubscription(security.SigningKey("sub-key"), 60, []string{"/buckets/uploads"})
|
||||
ctx, cancel := context.WithCancel(metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer "+string(token))))
|
||||
defer cancel()
|
||||
stream := &fakeSubscribeStream{ctx: ctx}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- h.fs.SubscribeLocalMetadata(&filer_pb.SubscribeMetadataRequest{
|
||||
ClientName: "scope-test", ClientId: 9, ClientEpoch: 1, PathPrefix: "/",
|
||||
ClientSupportsMetadataChunks: true,
|
||||
}, stream)
|
||||
}()
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
for _, e := range []*filer_pb.LogEntry{
|
||||
scopedLogEntry(ts(8), "/", "outside-live", "", false),
|
||||
scopedLogEntry(ts(7), "/buckets/uploads", "in-live", "", false),
|
||||
} {
|
||||
if err := h.f.LocalMetaLogBuffer.AddLogEntryToBuffer(e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
want := "in-flushed,in-live,in-memory"
|
||||
var got []string
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
got = got[:0]
|
||||
for _, m := range stream.snapshot() {
|
||||
if enforce && len(m.LogFileRefs) > 0 {
|
||||
t.Fatal("a scoped subscriber was sent log file references")
|
||||
}
|
||||
for _, e := range append([]*filer_pb.SubscribeMetadataResponse{m}, m.Events...) {
|
||||
if n := e.GetEventNotification(); n.GetNewEntry() != nil {
|
||||
got = append(got, n.NewEntry.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (enforce && len(got) >= 3) || (!enforce && strings.Contains(strings.Join(got, ","), "outside-live")) {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
sort.Strings(got)
|
||||
if enforce && strings.Join(got, ",") != want {
|
||||
t.Fatalf("delivered %v, want %s", got, want)
|
||||
}
|
||||
if !enforce && !strings.Contains(strings.Join(got, ","), "outside-live") {
|
||||
t.Fatalf("without the subscribe key every event must still arrive; got %v", got)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("subscription did not end on cancel")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A subscription token opens no filer HTTP read or write, also when an
|
||||
// operator reuses the read or write key as the subscribe key.
|
||||
func TestMetadataSubscriptionTokenRefusedOverHTTP(t *testing.T) {
|
||||
for _, keys := range []struct{ write, read, subscribe string }{
|
||||
{"write-key", "read-key", "sub-key"},
|
||||
{"write-key", "same", "same"},
|
||||
{"same", "read-key", "same"},
|
||||
{"same", "same", "same"},
|
||||
} {
|
||||
fs := &FilerServer{filerGuard: security.NewGuard(nil, keys.write, 0, keys.read, 0)}
|
||||
token := string(security.GenJwtForMetadataSubscription(security.SigningKey(keys.subscribe), 60, []string{"/buckets/uploads"}))
|
||||
for _, c := range []struct {
|
||||
method string
|
||||
isWrite bool
|
||||
}{{"GET", false}, {"HEAD", false}, {"PUT", true}, {"POST", true}, {"DELETE", true}} {
|
||||
for _, target := range []string{"/buckets/uploads/a.txt", "/buckets/uploads/", "/etc/iam/identity.json"} {
|
||||
r := httptest.NewRequest(c.method, target, nil)
|
||||
r.Header.Set("Authorization", "Bearer "+token)
|
||||
if fs.maybeCheckJwtAuthorization(r, c.isWrite) {
|
||||
t.Errorf("keys %+v: %s %s accepted a subscription token", keys, c.method, target)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func init() {
|
||||
Commands = append(Commands, &commandFsMetaToken{})
|
||||
}
|
||||
|
||||
type commandFsMetaToken struct {
|
||||
}
|
||||
|
||||
func (c *commandFsMetaToken) Name() string {
|
||||
return "fs.meta.token"
|
||||
}
|
||||
|
||||
func (c *commandFsMetaToken) Help() string {
|
||||
return `mint a read-only token for metadata subscriptions under path prefixes
|
||||
|
||||
fs.meta.token -prefix=/buckets/uploads -expire=720h
|
||||
fs.meta.token -prefix=/buckets/a,/buckets/b
|
||||
|
||||
The token is signed with jwt.filer_signing.subscribe.key from security.toml.
|
||||
Send it as "authorization: Bearer <token>" gRPC metadata on SubscribeMetadata
|
||||
or SubscribeLocalMetadata; the filer then sends only events whose paths are
|
||||
all under one of the prefixes.
|
||||
|
||||
`
|
||||
}
|
||||
|
||||
func (c *commandFsMetaToken) HasTag(CommandTag) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *commandFsMetaToken) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
|
||||
tokenCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
|
||||
prefixes := tokenCommand.String("prefix", "", "comma-separated absolute path prefixes")
|
||||
expire := tokenCommand.Duration("expire", 24*time.Hour, "token lifetime")
|
||||
if err = tokenCommand.Parse(args); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
key := util.GetViper().GetString("jwt.filer_signing.subscribe.key")
|
||||
if key == "" {
|
||||
return fmt.Errorf("jwt.filer_signing.subscribe.key is not set")
|
||||
}
|
||||
expireSeconds := int(expire.Seconds())
|
||||
if expireSeconds <= 0 {
|
||||
return fmt.Errorf("-expire must be at least one second")
|
||||
}
|
||||
var prefixList []string
|
||||
for _, p := range strings.Split(*prefixes, ",") {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
prefixList = append(prefixList, p)
|
||||
}
|
||||
}
|
||||
token := security.GenJwtForMetadataSubscription(security.SigningKey(key), expireSeconds, prefixList)
|
||||
if token == "" {
|
||||
return fmt.Errorf("-prefix needs one or more absolute paths")
|
||||
}
|
||||
fmt.Fprintln(writer, token)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc/metadata"
|
||||
)
|
||||
|
||||
func TestFsMetaToken(t *testing.T) {
|
||||
v := util.GetViper()
|
||||
defer v.Set("jwt.filer_signing.subscribe.key", "")
|
||||
c := &commandFsMetaToken{}
|
||||
var out bytes.Buffer
|
||||
if err := c.Do([]string{"-prefix=/a"}, nil, &out); err == nil {
|
||||
t.Fatal("minted without a subscribe key")
|
||||
}
|
||||
|
||||
v.Set("jwt.filer_signing.subscribe.key", "sub-key")
|
||||
if err := c.Do([]string{"-prefix=/a", "-expire=soon"}, nil, &out); err == nil {
|
||||
t.Fatal("an unparsable flag must fail")
|
||||
}
|
||||
if err := c.Do([]string{"-prefix=a"}, nil, &out); err == nil {
|
||||
t.Fatal("minted for a relative prefix")
|
||||
}
|
||||
if err := c.Do([]string{"-prefix=/buckets/a, /buckets/b/", "-expire=1h"}, nil, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer "+strings.TrimSpace(out.String())))
|
||||
scope, err := security.CheckGrpcMetadataSubscription(ctx, security.SigningKey("sub-key"), nil, security.MethodSubscribeMetadata)
|
||||
if err != nil || strings.Join(scope.Prefixes, ",") != "/buckets/a,/buckets/b" || time.Until(scope.ExpiresAt) < 59*time.Minute {
|
||||
t.Fatalf("scope=%v err=%v", scope, err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user