mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 03:13:53 +01:00
* security: scoped read-only token for filer metadata subscriptions A token signed with jwt.filer_signing.subscribe.key names the subscribe methods and the path prefixes its holder may see. mTLS peers and admin tokens stay unrestricted. Internal subscribers attach an admin token when both keys are configured. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: enforce subscription token scope on metadata streams With jwt.filer_signing.subscribe.key set, SubscribeMetadata and SubscribeLocalMetadata authorize the caller first. A scoped caller gets a stream that sends only events whose paths are all under its prefixes, filters batches, and never carries log file refs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: send an admin token on internal metadata subscriptions Peer filers, metadata followers and the lifecycle reader attach an admin token when the subscribe key is configured, so they keep the full stream without mTLS. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * shell: add fs.meta.token to mint metadata subscription tokens Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: end scoped metadata subscriptions at token expiry The scope now carries the token's exp. The scoped stream's context ends at that time and the call returns Unauthenticated, so an open stream no longer outlives its token. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer.sync: sign metadata subscriptions with the source cluster's key MetadataFollowOption takes a FilerJwtProvider; filer.sync passes the one loaded from -a.security or -b.security, so clusters with different admin keys both accept the subscription. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * shell: report fs.meta.token flag errors Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: give scoped metadata subscribers their own client id A scoped caller could reuse another subscriber's ClientId with a higher epoch and replace that registration, disconnecting it. Scoped streams now register under a server-chosen id that no other subscriber holds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * security: accept only HS256 metadata subscription tokens Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * security: test HS384 metadata subscription token rejection Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: never let a scoped subscriber replace another registration The server-chosen id can be taken by another subscriber between choosing and registering it. Registering scoped streams with the lowest epoch makes addClient refuse them rather than replace the other registration. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: send position markers for events a scoped token withholds Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test a scoped stream withholding a whole event batch Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: filter SendMsg on scoped metadata streams The scoped wrappers only overrode Send, so a SendMsg on the embedded stream would reach the client unfiltered. Route it through the same filter and refuse any other message type. Ported from #12790. Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test a scoped subscription through the real subscribe loop Run SubscribeLocalMetadata with a scoped token over a flushed backlog, the in-memory tail and live events: only in-scope events arrive, sibling, "..", encoded and cross-boundary rename paths are dropped, and no log file references are sent although the client asks for them. Without the subscribe key every event still arrives. Ported from #12790. Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test that the HTTP API refuses metadata subscription tokens GET, HEAD, PUT, POST and DELETE all refuse a subscription token, also when the subscribe key equals the read or write key. Ported from #12790. Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mindaeon <30555587+mindaeon@users.noreply.github.com>
300 lines
11 KiB
Go
300 lines
11 KiB
Go
package pb
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/security"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
"google.golang.org/grpc"
|
|
)
|
|
|
|
type EventErrorType int
|
|
|
|
const (
|
|
TrivialOnError EventErrorType = iota
|
|
FatalOnError
|
|
RetryForeverOnError
|
|
DontLogError
|
|
)
|
|
|
|
// MetadataFollowOption is used to control the behavior of the metadata following
|
|
// process. Part of it is used as a cursor to resume the following process.
|
|
type MetadataFollowOption struct {
|
|
ClientName string
|
|
ClientId int32
|
|
ClientEpoch int32
|
|
SelfSignature int32
|
|
PathPrefix string
|
|
AdditionalPathPrefixes []string
|
|
DirectoriesToWatch []string
|
|
StartTsNs int64
|
|
StopTsNs int64
|
|
EventErrorType EventErrorType
|
|
// LogFileReaderFn, when non-nil, enables metadata chunks mode:
|
|
// the server sends log file chunk fids instead of streaming events,
|
|
// and the client reads directly from volume servers.
|
|
LogFileReaderFn LogFileReaderFn
|
|
// FilerJwtProvider, when non-nil, signs the subscription with the source
|
|
// cluster's own filer key instead of this process's configuration.
|
|
FilerJwtProvider security.FilerJwtProvider
|
|
// OnIdleHeartbeat, when non-nil, opts in to idle heartbeats: while the
|
|
// subscriber is caught up the server periodically sends an empty response
|
|
// carrying the current time, and this is called with that timestamp. It is
|
|
// a freshness signal only and does not advance StartTsNs, so the resume
|
|
// checkpoint stays on the last real event.
|
|
OnIdleHeartbeat func(tsNs int64)
|
|
// GetResumeTsNs, when non-nil, supplies the reconnect position instead of
|
|
// StartTsNs. It is read on every subscribe, so a callback can return the
|
|
// durably processed watermark while StartTsNs keeps tracking positions the
|
|
// stream has merely seen.
|
|
GetResumeTsNs func() int64
|
|
// Resubscribe, when closed, drops the stream with ErrResubscribe so the
|
|
// caller's reconnect loop resubscribes from GetResumeTsNs and replays the
|
|
// events still pinning the processed watermark. Target-side job failures
|
|
// never surface on this stream, so without it a pinned event waits for an
|
|
// unrelated source-stream reconnect (or a restart) to be replayed.
|
|
Resubscribe <-chan struct{}
|
|
}
|
|
|
|
// ErrResubscribe ends a metadata follow when the consumer asked for the
|
|
// stream to drop so the reconnect replays what the resume watermark pins.
|
|
var ErrResubscribe = errors.New("resubscribe to replay events behind the failed offset")
|
|
|
|
type ProcessMetadataFunc func(resp *filer_pb.SubscribeMetadataResponse) error
|
|
|
|
func FollowMetadata(filerAddress ServerAddress, grpcDialOption grpc.DialOption, option *MetadataFollowOption, processEventFn ProcessMetadataFunc) error {
|
|
|
|
err := WithFilerClient(true, option.SelfSignature, filerAddress, grpcDialOption, makeSubscribeMetadataFunc(option, processEventFn))
|
|
if err != nil {
|
|
return fmt.Errorf("subscribing filer meta change: %w", err)
|
|
}
|
|
return err
|
|
}
|
|
|
|
func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient, option *MetadataFollowOption, processEventFn ProcessMetadataFunc) error {
|
|
|
|
err := filerClient.WithFilerClient(true, makeSubscribeMetadataFunc(option, processEventFn))
|
|
if err != nil {
|
|
return fmt.Errorf("subscribing filer meta change: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func makeSubscribeMetadataFunc(option *MetadataFollowOption, processEventFn ProcessMetadataFunc) func(client filer_pb.SeaweedFilerClient) error {
|
|
return func(client filer_pb.SeaweedFilerClient) error {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
sinceNs := option.StartTsNs
|
|
if option.GetResumeTsNs != nil {
|
|
sinceNs = option.GetResumeTsNs()
|
|
}
|
|
stream, err := client.SubscribeMetadata(security.WithMetadataSubscriptionCredentials(ctx, option.FilerJwtProvider), &filer_pb.SubscribeMetadataRequest{
|
|
ClientName: option.ClientName,
|
|
PathPrefix: option.PathPrefix,
|
|
PathPrefixes: option.AdditionalPathPrefixes,
|
|
Directories: option.DirectoriesToWatch,
|
|
SinceNs: sinceNs,
|
|
Signature: option.SelfSignature,
|
|
ClientId: option.ClientId,
|
|
ClientEpoch: option.ClientEpoch,
|
|
UntilNs: option.StopTsNs,
|
|
ClientSupportsBatching: true,
|
|
ClientSupportsMetadataChunks: option.LogFileReaderFn != nil,
|
|
ClientSupportsIdleHeartbeat: option.OnIdleHeartbeat != nil,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("subscribe: %w", err)
|
|
}
|
|
|
|
if option.Resubscribe != nil {
|
|
go func() {
|
|
select {
|
|
case <-option.Resubscribe:
|
|
cancel()
|
|
case <-ctx.Done():
|
|
}
|
|
}()
|
|
}
|
|
|
|
handleErr := func(resp *filer_pb.SubscribeMetadataResponse, err error) {
|
|
sourcePath := filer_pb.MetadataEventSourceFullPath(resp)
|
|
targetPath := filer_pb.MetadataEventTargetFullPath(resp)
|
|
switch option.EventErrorType {
|
|
case TrivialOnError:
|
|
glog.Errorf("process %s -> %s ts_ns=%d: %v", sourcePath, targetPath, resp.TsNs, err)
|
|
case FatalOnError:
|
|
glog.Fatalf("process %s -> %s ts_ns=%d: %v", sourcePath, targetPath, resp.TsNs, err)
|
|
case RetryForeverOnError:
|
|
waitTime := time.Second
|
|
for ctx.Err() == nil {
|
|
if err := processEventFn(resp); err == nil {
|
|
break
|
|
} else {
|
|
glog.Errorf("process %s -> %s ts_ns=%d: %v", sourcePath, targetPath, resp.TsNs, err)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-time.After(waitTime):
|
|
}
|
|
if waitTime < util.RetryWaitTime {
|
|
waitTime += waitTime / 2
|
|
}
|
|
}
|
|
case DontLogError:
|
|
// pass
|
|
default:
|
|
glog.Errorf("process %s -> %s ts_ns=%d: %v", sourcePath, targetPath, resp.TsNs, err)
|
|
}
|
|
}
|
|
|
|
// handleOneEvent processes a single response, whether it arrived as the
|
|
// batch envelope or inside resp.Events. Freshness signals carry a timestamp
|
|
// but no entry: the idle heartbeat (nil EventNotification) and the
|
|
// MaxUnsyncedEvents marker (empty one). Both route to OnIdleHeartbeat,
|
|
// else the marker pins sync_offset to the stale processed watermark and a
|
|
// nil heartbeat folded into a batch nil-derefs in the sync job path.
|
|
handleOneEvent := func(resp *filer_pb.SubscribeMetadataResponse) {
|
|
if resp.EventNotification == nil || filer_pb.IsEmpty(resp) {
|
|
if resp.TsNs > 0 && option.OnIdleHeartbeat != nil {
|
|
option.OnIdleHeartbeat(resp.TsNs)
|
|
}
|
|
// The marker advances the resume cursor past the filtered range; the
|
|
// heartbeat leaves it put so a restart cannot outrun a straggler. A
|
|
// consumer with a resume callback keeps its cursor in its processed
|
|
// watermark, so it sees the marker instead.
|
|
if resp.EventNotification != nil && resp.TsNs > 0 {
|
|
if option.GetResumeTsNs != nil {
|
|
if err := processEventFn(resp); err != nil {
|
|
handleErr(resp, err)
|
|
}
|
|
} else {
|
|
option.StartTsNs = resp.TsNs
|
|
}
|
|
}
|
|
return
|
|
}
|
|
if err := processEventFn(resp); err != nil {
|
|
handleErr(resp, err)
|
|
// RetryForeverOnError only returns once the event was handled;
|
|
// other modes leave it failed and the cursor stays behind it.
|
|
if option.EventErrorType != RetryForeverOnError {
|
|
return
|
|
}
|
|
}
|
|
if option.GetResumeTsNs == nil {
|
|
option.StartTsNs = resp.TsNs
|
|
}
|
|
}
|
|
|
|
var pendingRefs []*filer_pb.LogFileChunkRef
|
|
|
|
// drainPendingRefs reads whatever chunk refs have accumulated. The server
|
|
// sends refs in their own responses, so they can only be read once
|
|
// something tells us the run of refs has ended — either a normal event, or
|
|
// the end of the stream. A bounded subscription (StopTsNs set, range
|
|
// already in the past) may get nothing but refs and then EOF, so draining
|
|
// only on the former silently returns no events at all.
|
|
drainPendingRefs := func() error {
|
|
if len(pendingRefs) == 0 || option.LogFileReaderFn == nil {
|
|
return nil
|
|
}
|
|
readFromNs := option.StartTsNs
|
|
var progressFn func(tsNs int64)
|
|
if option.GetResumeTsNs != nil {
|
|
// The resume cursor lives in the processed watermark, so a
|
|
// resubscribed ref replay must not be filtered by positions the
|
|
// previous stream had only seen.
|
|
readFromNs = sinceNs
|
|
} else {
|
|
// Checkpoint each settled entry so a dropped stream resumes the
|
|
// replay where it stopped instead of re-reading the whole set.
|
|
progressFn = func(tsNs int64) {
|
|
if tsNs > option.StartTsNs {
|
|
option.StartTsNs = tsNs
|
|
}
|
|
}
|
|
}
|
|
lastTs, readErr := ReadLogFileRefs(pendingRefs, option.LogFileReaderFn,
|
|
readFromNs, option.StopTsNs,
|
|
PathFilter{
|
|
PathPrefix: option.PathPrefix,
|
|
AdditionalPathPrefixes: option.AdditionalPathPrefixes,
|
|
DirectoriesToWatch: option.DirectoriesToWatch,
|
|
},
|
|
processEventFn, progressFn)
|
|
if readErr != nil {
|
|
return fmt.Errorf("%w: %w", ErrLogFileRead, readErr)
|
|
}
|
|
// lastTs is the last settled entry's own timestamp; never rewind
|
|
// below the progress checkpoint it may lag.
|
|
if option.GetResumeTsNs == nil && lastTs > option.StartTsNs {
|
|
option.StartTsNs = lastTs
|
|
}
|
|
pendingRefs = nil
|
|
return nil
|
|
}
|
|
|
|
for {
|
|
resp, listenErr := stream.Recv()
|
|
if listenErr == io.EOF {
|
|
return drainPendingRefs()
|
|
}
|
|
if listenErr != nil {
|
|
select {
|
|
case <-option.Resubscribe:
|
|
return ErrResubscribe
|
|
default:
|
|
}
|
|
return listenErr
|
|
}
|
|
|
|
// Accumulate log file chunk references (metadata chunks mode)
|
|
if len(resp.LogFileRefs) > 0 {
|
|
pendingRefs = append(pendingRefs, resp.LogFileRefs...)
|
|
continue
|
|
}
|
|
|
|
// Process accumulated refs before handling normal events (transition point)
|
|
if err := drainPendingRefs(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Process the envelope event (top-level fields) and any batched tail.
|
|
// The server folds a backlog into one response: the first event lives in
|
|
// the top-level fields, the rest in resp.Events. Either slot can hold a
|
|
// freshness signal, so both go through the same handler.
|
|
handleOneEvent(resp)
|
|
for _, batchedEvent := range resp.Events {
|
|
handleOneEvent(batchedEvent)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func AddOffsetFunc(processEventFn ProcessMetadataFunc, offsetInterval time.Duration, offsetFunc func(counter int64, offset int64) error) ProcessMetadataFunc {
|
|
var counter int64
|
|
var lastWriteTime = time.Now()
|
|
return func(resp *filer_pb.SubscribeMetadataResponse) error {
|
|
if err := processEventFn(resp); err != nil {
|
|
return err
|
|
}
|
|
counter++
|
|
if lastWriteTime.Add(offsetInterval).Before(time.Now()) {
|
|
lastWriteTime = time.Now()
|
|
if err := offsetFunc(counter, resp.TsNs); err != nil {
|
|
return err
|
|
}
|
|
counter = 0
|
|
}
|
|
return nil
|
|
}
|
|
|
|
}
|