diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go index caaaddc502..cf7d210668 100644 --- a/weed/command/filer_sync.go +++ b/weed/command/filer_sync.go @@ -495,6 +495,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi StartTsNs: sourceFilerOffsetTsNs, StopTsNs: 0, EventErrorType: pb.RetryForeverOnError, + FilerJwtProvider: sourceJwtProvider, GetResumeTsNs: func() int64 { return resumeTsNsFor(processor) }, diff --git a/weed/command/scaffold/security.toml b/weed/command/scaffold/security.toml index a6b5231b6d..eda89c30a0 100644 --- a/weed/command/scaffold/security.toml +++ b/weed/command/scaffold/security.toml @@ -11,6 +11,7 @@ # The JWT signing keys map to: # WEED_JWT_SIGNING_KEY, WEED_JWT_SIGNING_READ_KEY (master <-> volume) # WEED_JWT_FILER_SIGNING_KEY, WEED_JWT_FILER_SIGNING_READ_KEY (s3/clients <-> filer) +# WEED_JWT_FILER_SIGNING_SUBSCRIBE_KEY (filer metadata subscriptions) # A set env var wins over the file, so the file can stay empty (or absent). # comma separated origins allowed to make requests to the filer and s3 gateway. @@ -75,6 +76,18 @@ expires_after_seconds = 10 # seconds key = "" expires_after_seconds = 10 # seconds +# Optional. If this key is set, the filer's SubscribeMetadata and +# SubscribeLocalMetadata gRPC calls require an mTLS-verified peer, an admin +# token signed with jwt.filer_signing.key, or a read-only token signed with this +# key. That token is an HS256 JWT with "exp", "allowed_methods" (a subset of +# SubscribeMetadata and SubscribeLocalMetadata) and "allowed_prefixes" +# (absolute paths); its holder only receives events whose paths are all under +# one of the prefixes. Mint one with "fs.meta.token" in weed shell. Without +# mTLS, set jwt.filer_signing.key and this key on every filer, S3 gateway, +# mount and filer.sync too, so they keep full access. +[jwt.filer_signing.subscribe] +key = "" + # gRPC mTLS configuration # All gRPC TLS authentications are mutual (mTLS) # The values for ca, cert, and key are paths to the certificate/key files diff --git a/weed/filer/meta_aggregator.go b/weed/filer/meta_aggregator.go index 83fe9db3c5..4494bdf2ac 100644 --- a/weed/filer/meta_aggregator.go +++ b/weed/filer/meta_aggregator.go @@ -12,6 +12,7 @@ import ( "time" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" "google.golang.org/grpc" @@ -724,7 +725,7 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress, return NewChunkStreamReaderFromLookup(ctx, lookupFn, chunks), nil } - stream, err := client.SubscribeLocalMetadata(ctx, &filer_pb.SubscribeMetadataRequest{ + stream, err := client.SubscribeLocalMetadata(security.WithMetadataSubscriptionCredentials(ctx, nil), &filer_pb.SubscribeMetadataRequest{ ClientName: "filer:" + string(self), PathPrefix: "/", SinceNs: lastTsNs, diff --git a/weed/pb/filer_pb_tail.go b/weed/pb/filer_pb_tail.go index 17123b036e..8fbb145419 100644 --- a/weed/pb/filer_pb_tail.go +++ b/weed/pb/filer_pb_tail.go @@ -9,6 +9,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" "google.golang.org/grpc" ) @@ -39,6 +40,9 @@ type MetadataFollowOption struct { // the server sends log file chunk fids instead of streaming events, // and the client reads directly from volume servers. LogFileReaderFn LogFileReaderFn + // FilerJwtProvider, when non-nil, signs the subscription with the source + // cluster's own filer key instead of this process's configuration. + FilerJwtProvider security.FilerJwtProvider // OnIdleHeartbeat, when non-nil, opts in to idle heartbeats: while the // subscriber is caught up the server periodically sends an empty response // carrying the current time, and this is called with that timestamp. It is @@ -91,7 +95,7 @@ func makeSubscribeMetadataFunc(option *MetadataFollowOption, processEventFn Proc if option.GetResumeTsNs != nil { sinceNs = option.GetResumeTsNs() } - stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{ + stream, err := client.SubscribeMetadata(security.WithMetadataSubscriptionCredentials(ctx, option.FilerJwtProvider), &filer_pb.SubscribeMetadataRequest{ ClientName: option.ClientName, PathPrefix: option.PathPrefix, PathPrefixes: option.AdditionalPathPrefixes, diff --git a/weed/s3api/s3lifecycle/reader/reader.go b/weed/s3api/s3lifecycle/reader/reader.go index c88badd789..aef41f73c8 100644 --- a/weed/s3api/s3lifecycle/reader/reader.go +++ b/weed/s3api/s3lifecycle/reader/reader.go @@ -11,6 +11,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle" + "github.com/seaweedfs/seaweedfs/weed/security" ) // Event is one in-shard meta-log event delivered to the router. @@ -157,7 +158,7 @@ func (r *Reader) Run(ctx context.Context, client filer_pb.SeaweedFilerClient, cl // Own context: aborting the RPC is the only way to unblock Recv. streamCtx, cancelStream := context.WithCancel(ctx) defer cancelStream() - stream, err := client.SubscribeMetadata(streamCtx, &filer_pb.SubscribeMetadataRequest{ + stream, err := client.SubscribeMetadata(security.WithMetadataSubscriptionCredentials(streamCtx, nil), &filer_pb.SubscribeMetadataRequest{ ClientName: clientName, PathPrefix: r.BucketsPath, SinceNs: sinceNs, diff --git a/weed/security/metadata_subscribe.go b/weed/security/metadata_subscribe.go new file mode 100644 index 0000000000..2c13f9c2ce --- /dev/null +++ b/weed/security/metadata_subscribe.go @@ -0,0 +1,148 @@ +package security + +import ( + "context" + "path" + "strings" + "time" + + jwt "github.com/golang-jwt/jwt/v5" + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/util" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +// The filer gRPC methods a scoped subscription token may name in allowed_methods. +const ( + MethodSubscribeMetadata = "SubscribeMetadata" + MethodSubscribeLocalMetadata = "SubscribeLocalMetadata" +) + +// GenJwtForMetadataSubscription mints a read-only token for the filer's +// metadata subscription calls, limited to events under prefixes. It carries +// allowed_prefixes and allowed_methods, so it is never an admin token. +// Returns an empty string without a key, an expiry, or an absolute prefix. +func GenJwtForMetadataSubscription(signingKey SigningKey, expiresAfterSec int, prefixes []string) EncodedJwt { + prefixes, ok := cleanMetadataPrefixes(prefixes) + if len(signingKey) == 0 || expiresAfterSec <= 0 || !ok { + return "" + } + claims := SeaweedFilerClaims{ + AllowedPrefixes: prefixes, + AllowedMethods: []string{MethodSubscribeMetadata, MethodSubscribeLocalMetadata}, + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Second * time.Duration(expiresAfterSec))), + }, + } + encoded, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(signingKey)) + if err != nil { + glog.V(0).Infof("Failed to sign metadata subscription claims: %v", err) + return "" + } + return EncodedJwt(encoded) +} + +// cleanMetadataPrefixes cleans prefixes; ok is false when the list is empty +// or any prefix is not absolute. +func cleanMetadataPrefixes(prefixes []string) (cleaned []string, ok bool) { + for _, p := range prefixes { + if !strings.HasPrefix(p, "/") { + return nil, false + } + cleaned = append(cleaned, path.Clean(p)) + } + return cleaned, len(cleaned) > 0 +} + +// MetadataPathInScope reports whether fullPath is one of the prefixes or below +// one, by whole path segments: "/buckets/a" holds "/buckets/a/x", not "/buckets/ab". +func MetadataPathInScope(fullPath string, prefixes []string) bool { + fullPath = path.Clean("/" + fullPath) + for _, p := range prefixes { + if p == "/" || fullPath == p || strings.HasPrefix(fullPath, p+"/") { + return true + } + } + return false +} + +// MetadataSubscriptionScope limits a subscription to events under Prefixes +// until ExpiresAt. +type MetadataSubscriptionScope struct { + Prefixes []string + ExpiresAt time.Time +} + +func metadataSubscriptionScope(signingKey SigningKey, token EncodedJwt, method string) (*MetadataSubscriptionScope, bool) { + parsed, err := decodeJwt(signingKey, token, &SeaweedFilerClaims{}, jwt.WithValidMethods([]string{jwt.SigningMethodHS256.Alg()})) + if err != nil || parsed == nil || !parsed.Valid { + return nil, false + } + claims, ok := parsed.Claims.(*SeaweedFilerClaims) + if !ok || claims.SessionId != "" || claims.ExpiresAt == nil { + return nil, false + } + allowsMethod := false + for _, m := range claims.AllowedMethods { + if m != MethodSubscribeMetadata && m != MethodSubscribeLocalMetadata { + return nil, false + } + allowsMethod = allowsMethod || m == method + } + if !allowsMethod { + return nil, false + } + prefixes, ok := cleanMetadataPrefixes(claims.AllowedPrefixes) + if !ok { + return nil, false + } + return &MetadataSubscriptionScope{Prefixes: prefixes, ExpiresAt: claims.ExpiresAt.Time}, true +} + +// CheckGrpcMetadataSubscription authorizes a metadata subscription call. +// Without subscribeKey every caller is unrestricted (nil scope). With it, an +// mTLS-verified peer or an admin token signed by adminKey is unrestricted, +// and a token signed by subscribeKey is limited to the returned scope. +func CheckGrpcMetadataSubscription(ctx context.Context, subscribeKey, adminKey SigningKey, method string) (scope *MetadataSubscriptionScope, err error) { + if len(subscribeKey) == 0 || isMtlsVerifiedPeer(ctx) { + return nil, nil + } + md, _ := metadata.FromIncomingContext(ctx) + authHeaders := md.Get("authorization") + if len(authHeaders) > maxGrpcAdminAuthValues { + authHeaders = authHeaders[:maxGrpcAdminAuthValues] + } + for _, raw := range authHeaders { + parts := strings.Fields(raw) + if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") || parts[1] == "" { + continue + } + token := EncodedJwt(parts[1]) + if IsFilerAdminToken(adminKey, token) { + return nil, nil + } + if scope, ok := metadataSubscriptionScope(subscribeKey, token, method); ok { + return scope, nil + } + } + return nil, status.Error(codes.Unauthenticated, "metadata subscription requires mTLS, an admin token, or a subscription token for this method") +} + +// WithMetadataSubscriptionCredentials adds an admin token to a subscription +// call: from jwtProvider when the caller has per-cluster keys, else when this +// process has both jwt.filer_signing.key and jwt.filer_signing.subscribe.key, +// so internal subscribers keep full access. +func WithMetadataSubscriptionCredentials(ctx context.Context, jwtProvider FilerJwtProvider) context.Context { + var token EncodedJwt + if jwtProvider != nil { + token = jwtProvider(true) + } else if v := util.GetViper(); v.GetString("jwt.filer_signing.subscribe.key") != "" { + token = GenJwtForFilerAdmin(SigningKey(v.GetString("jwt.filer_signing.key")), v.GetInt("jwt.filer_signing.expires_after_seconds")) + } + if token == "" { + return ctx + } + return metadata.AppendToOutgoingContext(ctx, "authorization", BearerPrefix+string(token)) +} diff --git a/weed/security/metadata_subscribe_test.go b/weed/security/metadata_subscribe_test.go new file mode 100644 index 0000000000..c5b7a1e758 --- /dev/null +++ b/weed/security/metadata_subscribe_test.go @@ -0,0 +1,152 @@ +package security + +import ( + "context" + "slices" + "testing" + "time" + + jwt "github.com/golang-jwt/jwt/v5" + "github.com/seaweedfs/seaweedfs/weed/util" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +func incomingBearer(token string) context.Context { + return metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer "+token)) +} + +func TestCheckGrpcMetadataSubscription(t *testing.T) { + subKey := SigningKey("subscribe-key") + adminKey := SigningKey("admin-key") + scoped := string(GenJwtForMetadataSubscription(subKey, 60, []string{"/buckets/uploads", "/buckets/logs/"})) + if scoped == "" { + t.Fatal("no scoped token") + } + signWith := func(method jwt.SigningMethod, key SigningKey, claims jwt.Claims) string { + s, err := jwt.NewWithClaims(method, claims).SignedString([]byte(key)) + if err != nil { + t.Fatal(err) + } + return s + } + sign := func(key SigningKey, claims jwt.Claims) string { return signWith(jwt.SigningMethodHS256, key, claims) } + exp := jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Minute))} + subscribeOnly := []string{MethodSubscribeMetadata} + wantScope := []string{"/buckets/uploads", "/buckets/logs"} + + tests := []struct { + name string + ctx context.Context + subKey SigningKey + method string + wantScope []string + wantCode codes.Code + }{ + {"feature off keeps the call open", context.Background(), nil, MethodSubscribeMetadata, nil, codes.OK}, + {"scoped token", incomingBearer(scoped), subKey, MethodSubscribeMetadata, wantScope, codes.OK}, + {"scoped token, local call", incomingBearer(scoped), subKey, MethodSubscribeLocalMetadata, wantScope, codes.OK}, + {"admin token is unrestricted", incomingBearer(string(GenJwtForFilerAdmin(adminKey, 60))), subKey, MethodSubscribeMetadata, nil, codes.OK}, + {"no token", context.Background(), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated}, + {"scoped token signed with another key", incomingBearer(string(GenJwtForMetadataSubscription(SigningKey("other"), 60, []string{"/a"}))), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated}, + {"scoped token signed with the admin key", incomingBearer(string(GenJwtForMetadataSubscription(adminKey, 60, []string{"/a"}))), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated}, + {"admin token signed with the subscribe key", incomingBearer(string(GenJwtForFilerAdmin(subKey, 60))), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated}, + {"token without prefixes", incomingBearer(sign(subKey, SeaweedFilerClaims{AllowedMethods: subscribeOnly, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated}, + {"token without methods", incomingBearer(sign(subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a"}, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated}, + {"token for a write method too", incomingBearer(sign(subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a"}, AllowedMethods: []string{MethodSubscribeMetadata, "PUT"}, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated}, + {"token for the other method only", incomingBearer(sign(subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a"}, AllowedMethods: []string{MethodSubscribeLocalMetadata}, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated}, + {"relative prefix", incomingBearer(sign(subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a", "b"}, AllowedMethods: subscribeOnly, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated}, + {"session token", incomingBearer(sign(subKey, SeaweedFilerClaims{SessionId: "s", AllowedPrefixes: []string{"/a"}, AllowedMethods: subscribeOnly, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated}, + {"token without expiry", incomingBearer(sign(subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a"}, AllowedMethods: subscribeOnly})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated}, + {"HS384 token", incomingBearer(signWith(jwt.SigningMethodHS384, subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a"}, AllowedMethods: subscribeOnly, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated}, + {"HS512 token", incomingBearer(signWith(jwt.SigningMethodHS512, subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a"}, AllowedMethods: subscribeOnly, RegisteredClaims: exp})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated}, + {"expired token", incomingBearer(sign(subKey, SeaweedFilerClaims{AllowedPrefixes: []string{"/a"}, AllowedMethods: subscribeOnly, RegisteredClaims: jwt.RegisteredClaims{ExpiresAt: jwt.NewNumericDate(time.Now().Add(-time.Hour))}})), subKey, MethodSubscribeMetadata, nil, codes.Unauthenticated}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scope, err := CheckGrpcMetadataSubscription(tt.ctx, tt.subKey, adminKey, tt.method) + if status.Code(err) != tt.wantCode { + t.Fatalf("code = %v (%v), want %v", status.Code(err), err, tt.wantCode) + } + if (scope == nil) != (tt.wantScope == nil) || scope != nil && !slices.Equal(scope.Prefixes, tt.wantScope) { + t.Fatalf("scope = %+v, want %v", scope, tt.wantScope) + } + if scope != nil && time.Until(scope.ExpiresAt) <= 0 { + t.Fatalf("scope expiry %v is not in the future", scope.ExpiresAt) + } + }) + } +} + +func TestGenJwtForMetadataSubscriptionRefusesUnusableScope(t *testing.T) { + key := SigningKey("k") + for name, tok := range map[string]EncodedJwt{ + "no expiry": GenJwtForMetadataSubscription(key, 0, []string{"/a"}), + "no prefix": GenJwtForMetadataSubscription(key, 60, nil), + "relative prefix": GenJwtForMetadataSubscription(key, 60, []string{"/a", "b"}), + "no key": GenJwtForMetadataSubscription(nil, 60, []string{"/a"}), + } { + if tok != "" { + t.Errorf("%s: minted %q", name, tok) + } + } +} + +func TestScopedSubscriptionTokenIsNotAdmin(t *testing.T) { + key := SigningKey("same-key") + if IsFilerAdminToken(key, GenJwtForMetadataSubscription(key, 60, []string{"/a"})) { + t.Fatal("a scoped subscription token must not count as an admin token") + } +} + +func TestMetadataPathInScope(t *testing.T) { + prefixes := []string{"/buckets/a"} + for p, want := range map[string]bool{ + "/buckets/a": true, + "/buckets/a/x/y": true, + "/buckets/ab": false, + "/buckets": false, + "/buckets/a/../b/x": false, + "/buckets/a/../a/x": true, + "/": false, + } { + if got := MetadataPathInScope(p, prefixes); got != want { + t.Errorf("MetadataPathInScope(%q) = %v, want %v", p, got, want) + } + } + if !MetadataPathInScope("/anything", []string{"/"}) { + t.Error("the root prefix holds every path") + } +} + +func TestWithMetadataSubscriptionCredentials(t *testing.T) { + v := util.GetViper() + defer v.Set("jwt.filer_signing.key", "") + defer v.Set("jwt.filer_signing.subscribe.key", "") + + v.Set("jwt.filer_signing.key", "admin-key") + if md, _ := metadata.FromOutgoingContext(WithMetadataSubscriptionCredentials(context.Background(), nil)); len(md.Get("authorization")) != 0 { + t.Fatal("no admin token without the subscribe key") + } + + v.Set("jwt.filer_signing.subscribe.key", "subscribe-key") + md, _ := metadata.FromOutgoingContext(WithMetadataSubscriptionCredentials(context.Background(), nil)) + in := metadata.NewIncomingContext(context.Background(), md) + scope, err := CheckGrpcMetadataSubscription(in, SigningKey("subscribe-key"), SigningKey("admin-key"), MethodSubscribeLocalMetadata) + if err != nil || scope != nil { + t.Fatalf("scope=%v err=%v, want unrestricted", scope, err) + } + + clusterKey := SigningKey("cluster-b-key") + md, _ = metadata.FromOutgoingContext(WithMetadataSubscriptionCredentials(context.Background(), func(bool) EncodedJwt { + return GenJwtForFilerServer(clusterKey, 10) + })) + if len(md.Get("authorization")) != 1 { + t.Fatalf("want only the per-cluster token, got %v", md.Get("authorization")) + } + scope, err = CheckGrpcMetadataSubscription(metadata.NewIncomingContext(context.Background(), md), SigningKey("subscribe-b"), clusterKey, MethodSubscribeMetadata) + if err != nil || scope != nil { + t.Fatalf("per-cluster token: scope=%v err=%v, want unrestricted", scope, err) + } +} diff --git a/weed/server/filer_grpc_server_sub_meta.go b/weed/server/filer_grpc_server_sub_meta.go index f96a6a1787..c9ac2fddd0 100644 --- a/weed/server/filer_grpc_server_sub_meta.go +++ b/weed/server/filer_grpc_server_sub_meta.go @@ -18,6 +18,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util/log_buffer" ) @@ -640,6 +641,18 @@ func (p *gapPass) park(ctx context.Context, cursor *log_buffer.MessagePosition, } func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, stream filer_pb.SeaweedFiler_SubscribeMetadataServer) (err error) { + scope, err := metadataSubscriptionScope(stream.Context(), security.MethodSubscribeMetadata) + if err != nil { + return err + } + if scope != nil { + req.ClientSupportsMetadataChunks = false + req.ClientId, req.ClientEpoch = fs.unusedClientId(), scopedClientEpoch + scoped, cancel := newScopedMetadataStream(stream.Context(), scope) + defer cancel() + defer func() { err = scoped.endErr(err) }() + stream = &scopedSubscribeStream{SeaweedFiler_SubscribeMetadataServer: stream, scopedMetadataStream: scoped} + } // A filer that has not learned remote peers yet serves the local log and // upgrades when the first one appears. RemotePeerArrivedChan takes the // arrival channel under the same lock as the peer check, so a peer @@ -1122,7 +1135,19 @@ func (fs *FilerServer) endOfSubscription(stream context.Context, err error) erro return err } -func (fs *FilerServer) SubscribeLocalMetadata(req *filer_pb.SubscribeMetadataRequest, stream filer_pb.SeaweedFiler_SubscribeLocalMetadataServer) error { +func (fs *FilerServer) SubscribeLocalMetadata(req *filer_pb.SubscribeMetadataRequest, stream filer_pb.SeaweedFiler_SubscribeLocalMetadataServer) (err error) { + scope, err := metadataSubscriptionScope(stream.Context(), security.MethodSubscribeLocalMetadata) + if err != nil { + return err + } + if scope != nil { + req.ClientSupportsMetadataChunks = false + req.ClientId, req.ClientEpoch = fs.unusedClientId(), scopedClientEpoch + scoped, cancel := newScopedMetadataStream(stream.Context(), scope) + defer cancel() + defer func() { err = scoped.endErr(err) }() + stream = &scopedSubscribeLocalStream{SeaweedFiler_SubscribeLocalMetadataServer: stream, scopedMetadataStream: scoped} + } return fs.subscribeLocalMetadata(req, stream, nil) } diff --git a/weed/server/filer_grpc_server_sub_meta_scope.go b/weed/server/filer_grpc_server_sub_meta_scope.go new file mode 100644 index 0000000000..833a5d9537 --- /dev/null +++ b/weed/server/filer_grpc_server_sub_meta_scope.go @@ -0,0 +1,180 @@ +package weed_server + +import ( + "context" + "fmt" + "math" + "math/rand/v2" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/util" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func metadataSubscriptionScope(ctx context.Context, method string) (*security.MetadataSubscriptionScope, error) { + v := util.GetViper() + return security.CheckGrpcMetadataSubscription(ctx, + security.SigningKey(v.GetString("jwt.filer_signing.subscribe.key")), + security.SigningKey(v.GetString("jwt.filer_signing.key")), + method) +} + +// scopeMetadataResponse keeps only what a scoped subscriber may see: events +// whose every path is in scope, so a rename across the boundary never shows +// the far side, and no log file refs. Returns nil when nothing is left. +func scopeMetadataResponse(resp *filer_pb.SubscribeMetadataResponse, prefixes []string) *filer_pb.SubscribeMetadataResponse { + if resp.EventNotification == nil && len(resp.LogFileRefs) > 0 { + return nil + } + var kept []*filer_pb.SubscribeMetadataResponse + for _, e := range resp.Events { + if metadataEventInScope(e, prefixes) { + kept = append(kept, e) + } + } + headInScope := metadataEventInScope(resp, prefixes) + if !headInScope && len(kept) == 0 { + return nil + } + if headInScope && len(kept) == len(resp.Events) && len(resp.LogFileRefs) == 0 { + return resp + } + scoped := &filer_pb.SubscribeMetadataResponse{ + TsNs: resp.TsNs, + Events: kept, + FlushedTsNs: resp.FlushedTsNs, + } + if headInScope { + scoped.Directory = resp.Directory + scoped.EventNotification = resp.EventNotification + scoped.MutationTsNs = resp.MutationTsNs + } else { + // the batch head keeps its position as an empty marker + scoped.EventNotification = &filer_pb.EventNotification{} + } + return scoped +} + +func metadataEventInScope(event *filer_pb.SubscribeMetadataResponse, prefixes []string) bool { + if event.EventNotification == nil || filer_pb.IsEmpty(event) { + return true + } + return security.MetadataPathInScope(filer_pb.MetadataEventSourceFullPath(event), prefixes) && + security.MetadataPathInScope(filer_pb.MetadataEventTargetFullPath(event), prefixes) +} + +var errMetadataSubscriptionExpired = status.Error(codes.Unauthenticated, "metadata subscription token expired") + +// scopedMetadataStream ends its context when the token expires, so an open +// subscription stops there. +type scopedMetadataStream struct { + ctx context.Context + scope *security.MetadataSubscriptionScope + withheld *int64 +} + +// scopedClientEpoch is below every other epoch, so a scoped registration +// never replaces another one, even if its id gets taken after it was chosen. +const scopedClientEpoch = math.MinInt32 + +// unusedClientId gives a scoped caller a registration id of its own, so it +// cannot replace or squat on another subscriber's registration. +func (fs *FilerServer) unusedClientId() int32 { + fs.knownListenersLock.Lock() + defer fs.knownListenersLock.Unlock() + for { + id := rand.Int32N(1<<31-1) + 1 + _, found := fs.knownListeners[id] + _, foundLocal := fs.knownListeners[-id] + if !found && !foundLocal { + return id + } + } +} + +func newScopedMetadataStream(parent context.Context, scope *security.MetadataSubscriptionScope) (scopedMetadataStream, context.CancelFunc) { + ctx, cancel := context.WithDeadline(parent, scope.ExpiresAt) + return scopedMetadataStream{ctx: ctx, scope: scope, withheld: new(int64)}, cancel +} + +func (s scopedMetadataStream) Context() context.Context { return s.ctx } + +func (s scopedMetadataStream) send(resp *filer_pb.SubscribeMetadataResponse, send func(*filer_pb.SubscribeMetadataResponse) error) error { + if !time.Now().Before(s.scope.ExpiresAt) { + return errMetadataSubscriptionExpired + } + if scoped := scopeMetadataResponse(resp, s.scope.Prefixes); scoped != nil { + if err := send(scoped); err != nil { + return err + } + *s.withheld = 0 + return nil + } + if resp.EventNotification == nil { + return nil + } + // withheld events still move the subscriber's resume position + *s.withheld += int64(1 + len(resp.Events)) + if *s.withheld <= MaxUnsyncedEvents { + return nil + } + tsNs := resp.TsNs + for _, e := range resp.Events { + tsNs = max(tsNs, e.TsNs) + } + if err := send(&filer_pb.SubscribeMetadataResponse{EventNotification: &filer_pb.EventNotification{}, TsNs: tsNs}); err != nil { + return err + } + *s.withheld = 0 + return nil +} + +// sendMsg filters SendMsg too, so the embedded stream never sends unfiltered. +func (s scopedMetadataStream) sendMsg(m any, send func(*filer_pb.SubscribeMetadataResponse) error) error { + resp, ok := m.(*filer_pb.SubscribeMetadataResponse) + if !ok { + return fmt.Errorf("scoped metadata stream: unexpected message %T", m) + } + return s.send(resp, send) +} + +// endErr reports an expired token in place of whatever ended the stream. +func (s scopedMetadataStream) endErr(err error) error { + if !time.Now().Before(s.scope.ExpiresAt) { + return errMetadataSubscriptionExpired + } + return err +} + +type scopedSubscribeStream struct { + filer_pb.SeaweedFiler_SubscribeMetadataServer + scopedMetadataStream +} + +func (s *scopedSubscribeStream) Context() context.Context { return s.ctx } + +func (s *scopedSubscribeStream) Send(resp *filer_pb.SubscribeMetadataResponse) error { + return s.send(resp, s.SeaweedFiler_SubscribeMetadataServer.Send) +} + +func (s *scopedSubscribeStream) SendMsg(m any) error { + return s.sendMsg(m, s.SeaweedFiler_SubscribeMetadataServer.Send) +} + +type scopedSubscribeLocalStream struct { + filer_pb.SeaweedFiler_SubscribeLocalMetadataServer + scopedMetadataStream +} + +func (s *scopedSubscribeLocalStream) Context() context.Context { return s.ctx } + +func (s *scopedSubscribeLocalStream) Send(resp *filer_pb.SubscribeMetadataResponse) error { + return s.send(resp, s.SeaweedFiler_SubscribeLocalMetadataServer.Send) +} + +func (s *scopedSubscribeLocalStream) SendMsg(m any) error { + return s.sendMsg(m, s.SeaweedFiler_SubscribeLocalMetadataServer.Send) +} diff --git a/weed/server/filer_grpc_server_sub_meta_scope_test.go b/weed/server/filer_grpc_server_sub_meta_scope_test.go new file mode 100644 index 0000000000..64301018a6 --- /dev/null +++ b/weed/server/filer_grpc_server_sub_meta_scope_test.go @@ -0,0 +1,395 @@ +package weed_server + +import ( + "context" + "net/http/httptest" + "sort" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/util" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +func scopeTestEvent(dir, name string, tsNs int64) *filer_pb.SubscribeMetadataResponse { + return &filer_pb.SubscribeMetadataResponse{ + Directory: dir, + TsNs: tsNs, + EventNotification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: name}}, + } +} + +func TestScopeMetadataResponse(t *testing.T) { + prefixes := []string{"/buckets/uploads"} + + if got := scopeMetadataResponse(scopeTestEvent("/buckets/uploads/2026", "a.txt", 1), prefixes); got == nil { + t.Error("an event under the prefix must pass") + } + if got := scopeMetadataResponse(scopeTestEvent("/buckets", "uploads", 1), prefixes); got == nil { + t.Error("an event on the prefix itself must pass") + } + for _, dir := range []string{"/buckets/other", "/buckets/uploads-private", "/buckets/uploads/../other"} { + if got := scopeMetadataResponse(scopeTestEvent(dir, "a.txt", 1), prefixes); got != nil { + t.Errorf("an event in %s must be dropped", dir) + } + } + + out := scopeTestEvent("/buckets/uploads", "a.txt", 1) + out.EventNotification.OldEntry = &filer_pb.Entry{Name: "a.txt"} + out.EventNotification.NewParentPath = "/buckets/other" + if got := scopeMetadataResponse(out, prefixes); got != nil { + t.Error("a rename out of the scope must be dropped") + } + in := scopeTestEvent("/buckets/other", "a.txt", 1) + in.EventNotification.OldEntry = &filer_pb.Entry{Name: "a.txt"} + in.EventNotification.NewParentPath = "/buckets/uploads" + if got := scopeMetadataResponse(in, prefixes); got != nil { + t.Error("a rename into the scope must be dropped") + } + del := &filer_pb.SubscribeMetadataResponse{Directory: "/etc", EventNotification: &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: "x"}}} + if got := scopeMetadataResponse(del, prefixes); got != nil { + t.Error("a delete outside the scope must be dropped") + } + + for name, marker := range map[string]*filer_pb.SubscribeMetadataResponse{ + "heartbeat": {TsNs: 5}, + "flush report": {FlushedTsNs: 5}, + "empty marker": {TsNs: 5, EventNotification: &filer_pb.EventNotification{}}, + } { + if got := scopeMetadataResponse(marker, prefixes); got != marker { + t.Errorf("%s must pass unchanged", name) + } + } + if got := scopeMetadataResponse(&filer_pb.SubscribeMetadataResponse{LogFileRefs: []*filer_pb.LogFileChunkRef{{}}}, prefixes); got != nil { + t.Error("log file refs must never be sent") + } + + batch := scopeTestEvent("/buckets/other", "x", 10) + batch.MutationTsNs = 9 + batch.FlushedTsNs = 7 + batch.Events = []*filer_pb.SubscribeMetadataResponse{scopeTestEvent("/buckets/uploads", "ok", 11), scopeTestEvent("/etc", "secret", 12)} + got := scopeMetadataResponse(batch, prefixes) + if got == nil || len(got.Events) != 1 || got.Events[0].Directory != "/buckets/uploads" { + t.Fatalf("batch not filtered: %+v", got) + } + if got.Directory != "" || !filer_pb.IsEmpty(got) || got.MutationTsNs != 0 || got.TsNs != 10 || got.FlushedTsNs != 7 { + t.Errorf("an out-of-scope batch head must become a marker at its position: %+v", got) + } + if len(batch.Events) != 2 { + t.Error("the original response must not be modified") + } + + inScopeHead := scopeTestEvent("/buckets/uploads", "head", 10) + inScopeHead.Events = []*filer_pb.SubscribeMetadataResponse{scopeTestEvent("/etc", "secret", 11)} + got = scopeMetadataResponse(inScopeHead, prefixes) + if got == nil || got.EventNotification != inScopeHead.EventNotification || len(got.Events) != 0 { + t.Errorf("an in-scope head keeps its event and drops the out-of-scope tail: %+v", got) + } +} + +func TestScopedSubscriptionEndsAtTokenExpiry(t *testing.T) { + h := newSubscribeHarness(t) + v := util.GetViper() + v.Set("jwt.filer_signing.subscribe.key", "sub-key") + t.Cleanup(func() { v.Set("jwt.filer_signing.subscribe.key", "") }) + token := security.GenJwtForMetadataSubscription(security.SigningKey("sub-key"), 2, []string{"/t"}) + ctx, cancel := context.WithCancel(metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer "+string(token)))) + defer cancel() + stream := &fakeSubscribeStream{ctx: ctx} + done := make(chan error, 1) + go func() { + done <- h.fs.SubscribeLocalMetadata(&filer_pb.SubscribeMetadataRequest{ClientName: "scoped", ClientId: 9, ClientEpoch: 1, SinceNs: 1}, stream) + }() + + now := time.Now().UnixNano() + h.append(now) + outside, _ := proto.Marshal(&filer_pb.SubscribeMetadataResponse{Directory: "/other", TsNs: now + 1, EventNotification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "x"}}}) + if err := h.f.LocalMetaLogBuffer.AddLogEntryToBuffer(&filer_pb.LogEntry{TsNs: now + 1, Data: outside, Key: []byte("/other/x")}); err != nil { + t.Fatal(err) + } + + select { + case err := <-done: + if status.Code(err) != codes.Unauthenticated { + t.Fatalf("stream ended with %v, want Unauthenticated at token expiry", err) + } + case <-time.After(5 * time.Second): + t.Fatal("scoped subscription outlived its token") + } + for _, m := range stream.snapshot() { + if m.Directory == "/other" { + t.Fatalf("out-of-scope event delivered: %+v", m) + } + } + if got := eventTimestamps(stream.snapshot()); len(got) != 1 || got[0] != now { + t.Fatalf("delivered %v, want only the in-scope event", got) + } +} + +func TestScopedSubscriptionKeepsOtherRegistrations(t *testing.T) { + h := newSubscribeHarness(t) + v := util.GetViper() + v.Set("jwt.filer_signing.subscribe.key", "sub-key") + t.Cleanup(func() { v.Set("jwt.filer_signing.subscribe.key", "") }) + if replacing, _, _ := h.fs.addClient("local", "mount", "10.0.0.1:1", "/", -9, 1); !replacing { + t.Fatal("unrestricted registration was not added") + } + + token := security.GenJwtForMetadataSubscription(security.SigningKey("sub-key"), 60, []string{"/t"}) + ctx, cancel := context.WithCancel(metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer "+string(token)))) + stream := &fakeSubscribeStream{ctx: ctx} + done := make(chan error, 1) + go func() { + done <- h.fs.SubscribeLocalMetadata(&filer_pb.SubscribeMetadataRequest{ClientName: "scoped", ClientId: 9, ClientEpoch: 100, SinceNs: 1}, stream) + }() + + now := time.Now().UnixNano() + h.append(now) + deadline := time.Now().Add(5 * time.Second) + for len(stream.snapshot()) == 0 { + if time.Now().After(deadline) { + t.Fatal("scoped subscription delivered nothing") + } + time.Sleep(10 * time.Millisecond) + } + if !h.fs.hasClient(-9, 1) { + t.Fatal("scoped caller replaced the unrestricted registration") + } + + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("scoped subscription did not end") + } + if !h.fs.hasClient(-9, 1) { + t.Fatal("scoped caller removed the unrestricted registration on exit") + } +} + +func TestScopedRegistrationNeverReplacesLaterRegistration(t *testing.T) { + h := newSubscribeHarness(t) + id := h.fs.unusedClientId() + if replacing, _, _ := h.fs.addClient("", "mount", "10.0.0.1:1", "/", id, 1); !replacing { + t.Fatal("unrestricted registration was not added") + } + if replacing, alreadyKnown, _ := h.fs.addClient("", "scoped", "10.0.0.2:1", "/t", id, scopedClientEpoch); replacing || !alreadyKnown { + t.Fatalf("scoped registration replacing=%v alreadyKnown=%v, want refused", replacing, alreadyKnown) + } + h.fs.deleteClient("", "scoped", id, scopedClientEpoch) + if !h.fs.hasClient(id, 1) { + t.Fatal("unrestricted registration lost to a scoped caller") + } +} + +func TestScopedStreamMarksWithheldProgress(t *testing.T) { + var sent []*filer_pb.SubscribeMetadataResponse + send := func(m *filer_pb.SubscribeMetadataResponse) error { + sent = append(sent, m) + return nil + } + s, cancel := newScopedMetadataStream(context.Background(), &security.MetadataSubscriptionScope{Prefixes: []string{"/a"}, ExpiresAt: time.Now().Add(time.Hour)}) + defer cancel() + var last int64 + for i := int64(1); i <= MaxUnsyncedEvents+1; i++ { + last = i + if err := s.send(scopeTestEvent("/other", "x", i), send); err != nil { + t.Fatal(err) + } + } + if len(sent) != 1 { + t.Fatalf("sent %d messages, want one position marker", len(sent)) + } + m := sent[0] + if m.TsNs != last || !filer_pb.IsEmpty(m) || m.Directory != "" || len(m.Events) > 0 { + t.Fatalf("marker %+v, want an empty event at %d", m, last) + } +} + +func TestScopedStreamMarksWithheldBatch(t *testing.T) { + var sent []*filer_pb.SubscribeMetadataResponse + send := func(m *filer_pb.SubscribeMetadataResponse) error { + sent = append(sent, m) + return nil + } + s, cancel := newScopedMetadataStream(context.Background(), &security.MetadataSubscriptionScope{Prefixes: []string{"/a"}, ExpiresAt: time.Now().Add(time.Hour)}) + defer cancel() + batch := scopeTestEvent("/other", "head", 1) + for i := int64(2); i <= MaxUnsyncedEvents+1; i++ { + batch.Events = append(batch.Events, scopeTestEvent("/other", "x", i)) + } + if err := s.send(batch, send); err != nil { + t.Fatal(err) + } + if len(sent) != 1 { + t.Fatalf("sent %d messages, want one position marker", len(sent)) + } + m, want := sent[0], int64(MaxUnsyncedEvents+1) + if m.TsNs != want || !filer_pb.IsEmpty(m) || m.Directory != "" || len(m.Events) > 0 { + t.Fatalf("marker %+v, want an empty event at %d", m, want) + } +} + +func TestScopedStreamSendMsgIsFiltered(t *testing.T) { + scoped, cancel := newScopedMetadataStream(context.Background(), &security.MetadataSubscriptionScope{Prefixes: []string{"/a"}, ExpiresAt: time.Now().Add(time.Hour)}) + defer cancel() + for _, s := range []interface{ SendMsg(any) error }{ + &scopedSubscribeStream{SeaweedFiler_SubscribeMetadataServer: &fakeSubscribeStream{ctx: context.Background()}, scopedMetadataStream: scoped}, + &scopedSubscribeLocalStream{SeaweedFiler_SubscribeLocalMetadataServer: &fakeSubscribeStream{ctx: context.Background()}, scopedMetadataStream: scoped}, + } { + if err := s.SendMsg(scopeTestEvent("/b", "x", 1)); err != nil { + t.Fatal(err) + } + if err := s.SendMsg(scopeTestEvent("/a", "x", 2)); err != nil { + t.Fatal(err) + } + if err := s.SendMsg(&filer_pb.Entry{}); err == nil { + t.Error("an unexpected message type must be refused") + } + var inner *fakeSubscribeStream + switch w := s.(type) { + case *scopedSubscribeStream: + inner = w.SeaweedFiler_SubscribeMetadataServer.(*fakeSubscribeStream) + case *scopedSubscribeLocalStream: + inner = w.SeaweedFiler_SubscribeLocalMetadataServer.(*fakeSubscribeStream) + } + if msgs := inner.snapshot(); len(msgs) != 1 || msgs[0].Directory != "/a" { + t.Errorf("%T sent %v", s, msgs) + } + } +} + +func scopedLogEntry(tsNs int64, dir, name, newParent string, oldEntry bool) *filer_pb.LogEntry { + n := &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: name}, NewParentPath: newParent} + if oldEntry { + n.OldEntry = &filer_pb.Entry{Name: name} + } + data, err := proto.Marshal(&filer_pb.SubscribeMetadataResponse{Directory: dir, EventNotification: n, TsNs: tsNs}) + if err != nil { + panic(err) + } + return &filer_pb.LogEntry{TsNs: tsNs, Data: data, Key: []byte(dir + "/" + name)} +} + +// The real subscribe loop with a scoped token: flushed backlog, the memory +// tail and live events are all filtered, and log file references are not sent +// although the client asks for them. +func TestSubscribeLocalMetadataScopedEndToEnd(t *testing.T) { + for _, enforce := range []bool{true, false} { + t.Run(map[bool]string{true: "scoped", false: "key unset"}[enforce], func(t *testing.T) { + h := newSubscribeHarness(t) + if enforce { + v := util.GetViper() + v.Set("jwt.filer_signing.subscribe.key", "sub-key") + t.Cleanup(func() { v.Set("jwt.filer_signing.subscribe.key", "") }) + } + + ts := func(i int) int64 { return h.base + int64(i)*int64(time.Millisecond) } + entries := []*filer_pb.LogEntry{ + scopedLogEntry(ts(0), "/buckets/uploads", "in-flushed", "", false), + scopedLogEntry(ts(1), "/buckets/uploads-private", "sibling", "", false), + scopedLogEntry(ts(2), "/etc/iam", "identity.json", "", false), + scopedLogEntry(ts(3), "/buckets/uploads/..", "climb", "", false), + scopedLogEntry(ts(4), "/buckets/uploads", "renamed-out", "/buckets/other", true), + scopedLogEntry(ts(5), "/buckets/uploads/2026", "in-memory", "", false), + scopedLogEntry(ts(6), "/buckets/uploads%2F..%2Fother", "encoded", "", false), + } + for i, e := range entries { + if err := h.f.LocalMetaLogBuffer.AddLogEntryToBuffer(e); err != nil { + t.Fatal(err) + } + if i == 4 { + h.f.LocalMetaLogBuffer.ForceFlush() + } + } + + token := security.GenJwtForMetadataSubscription(security.SigningKey("sub-key"), 60, []string{"/buckets/uploads"}) + ctx, cancel := context.WithCancel(metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer "+string(token)))) + defer cancel() + stream := &fakeSubscribeStream{ctx: ctx} + done := make(chan error, 1) + go func() { + done <- h.fs.SubscribeLocalMetadata(&filer_pb.SubscribeMetadataRequest{ + ClientName: "scope-test", ClientId: 9, ClientEpoch: 1, PathPrefix: "/", + ClientSupportsMetadataChunks: true, + }, stream) + }() + + time.Sleep(100 * time.Millisecond) + for _, e := range []*filer_pb.LogEntry{ + scopedLogEntry(ts(8), "/", "outside-live", "", false), + scopedLogEntry(ts(7), "/buckets/uploads", "in-live", "", false), + } { + if err := h.f.LocalMetaLogBuffer.AddLogEntryToBuffer(e); err != nil { + t.Fatal(err) + } + } + + want := "in-flushed,in-live,in-memory" + var got []string + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + got = got[:0] + for _, m := range stream.snapshot() { + if enforce && len(m.LogFileRefs) > 0 { + t.Fatal("a scoped subscriber was sent log file references") + } + for _, e := range append([]*filer_pb.SubscribeMetadataResponse{m}, m.Events...) { + if n := e.GetEventNotification(); n.GetNewEntry() != nil { + got = append(got, n.NewEntry.Name) + } + } + } + if (enforce && len(got) >= 3) || (!enforce && strings.Contains(strings.Join(got, ","), "outside-live")) { + break + } + time.Sleep(10 * time.Millisecond) + } + sort.Strings(got) + if enforce && strings.Join(got, ",") != want { + t.Fatalf("delivered %v, want %s", got, want) + } + if !enforce && !strings.Contains(strings.Join(got, ","), "outside-live") { + t.Fatalf("without the subscribe key every event must still arrive; got %v", got) + } + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("subscription did not end on cancel") + } + }) + } +} + +// A subscription token opens no filer HTTP read or write, also when an +// operator reuses the read or write key as the subscribe key. +func TestMetadataSubscriptionTokenRefusedOverHTTP(t *testing.T) { + for _, keys := range []struct{ write, read, subscribe string }{ + {"write-key", "read-key", "sub-key"}, + {"write-key", "same", "same"}, + {"same", "read-key", "same"}, + {"same", "same", "same"}, + } { + fs := &FilerServer{filerGuard: security.NewGuard(nil, keys.write, 0, keys.read, 0)} + token := string(security.GenJwtForMetadataSubscription(security.SigningKey(keys.subscribe), 60, []string{"/buckets/uploads"})) + for _, c := range []struct { + method string + isWrite bool + }{{"GET", false}, {"HEAD", false}, {"PUT", true}, {"POST", true}, {"DELETE", true}} { + for _, target := range []string{"/buckets/uploads/a.txt", "/buckets/uploads/", "/etc/iam/identity.json"} { + r := httptest.NewRequest(c.method, target, nil) + r.Header.Set("Authorization", "Bearer "+token) + if fs.maybeCheckJwtAuthorization(r, c.isWrite) { + t.Errorf("keys %+v: %s %s accepted a subscription token", keys, c.method, target) + } + } + } + } +} diff --git a/weed/shell/command_fs_meta_token.go b/weed/shell/command_fs_meta_token.go new file mode 100644 index 0000000000..a741a5d64c --- /dev/null +++ b/weed/shell/command_fs_meta_token.go @@ -0,0 +1,75 @@ +package shell + +import ( + "errors" + "flag" + "fmt" + "io" + "strings" + "time" + + "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +func init() { + Commands = append(Commands, &commandFsMetaToken{}) +} + +type commandFsMetaToken struct { +} + +func (c *commandFsMetaToken) Name() string { + return "fs.meta.token" +} + +func (c *commandFsMetaToken) Help() string { + return `mint a read-only token for metadata subscriptions under path prefixes + + fs.meta.token -prefix=/buckets/uploads -expire=720h + fs.meta.token -prefix=/buckets/a,/buckets/b + + The token is signed with jwt.filer_signing.subscribe.key from security.toml. + Send it as "authorization: Bearer " gRPC metadata on SubscribeMetadata + or SubscribeLocalMetadata; the filer then sends only events whose paths are + all under one of the prefixes. + +` +} + +func (c *commandFsMetaToken) HasTag(CommandTag) bool { + return false +} + +func (c *commandFsMetaToken) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + tokenCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + prefixes := tokenCommand.String("prefix", "", "comma-separated absolute path prefixes") + expire := tokenCommand.Duration("expire", 24*time.Hour, "token lifetime") + if err = tokenCommand.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } + return err + } + + key := util.GetViper().GetString("jwt.filer_signing.subscribe.key") + if key == "" { + return fmt.Errorf("jwt.filer_signing.subscribe.key is not set") + } + expireSeconds := int(expire.Seconds()) + if expireSeconds <= 0 { + return fmt.Errorf("-expire must be at least one second") + } + var prefixList []string + for _, p := range strings.Split(*prefixes, ",") { + if p = strings.TrimSpace(p); p != "" { + prefixList = append(prefixList, p) + } + } + token := security.GenJwtForMetadataSubscription(security.SigningKey(key), expireSeconds, prefixList) + if token == "" { + return fmt.Errorf("-prefix needs one or more absolute paths") + } + fmt.Fprintln(writer, token) + return nil +} diff --git a/weed/shell/command_fs_meta_token_test.go b/weed/shell/command_fs_meta_token_test.go new file mode 100644 index 0000000000..ddf26a11a7 --- /dev/null +++ b/weed/shell/command_fs_meta_token_test.go @@ -0,0 +1,39 @@ +package shell + +import ( + "bytes" + "context" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/util" + "google.golang.org/grpc/metadata" +) + +func TestFsMetaToken(t *testing.T) { + v := util.GetViper() + defer v.Set("jwt.filer_signing.subscribe.key", "") + c := &commandFsMetaToken{} + var out bytes.Buffer + if err := c.Do([]string{"-prefix=/a"}, nil, &out); err == nil { + t.Fatal("minted without a subscribe key") + } + + v.Set("jwt.filer_signing.subscribe.key", "sub-key") + if err := c.Do([]string{"-prefix=/a", "-expire=soon"}, nil, &out); err == nil { + t.Fatal("an unparsable flag must fail") + } + if err := c.Do([]string{"-prefix=a"}, nil, &out); err == nil { + t.Fatal("minted for a relative prefix") + } + if err := c.Do([]string{"-prefix=/buckets/a, /buckets/b/", "-expire=1h"}, nil, &out); err != nil { + t.Fatal(err) + } + ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer "+strings.TrimSpace(out.String()))) + scope, err := security.CheckGrpcMetadataSubscription(ctx, security.SigningKey("sub-key"), nil, security.MethodSubscribeMetadata) + if err != nil || strings.Join(scope.Prefixes, ",") != "/buckets/a,/buckets/b" || time.Until(scope.ExpiresAt) < 59*time.Minute { + t.Fatalf("scope=%v err=%v", scope, err) + } +}