mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 09:46:24 +01:00
* webhook: share the default target settings Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * webhook: managed webhook targets with a drainable queue Targets kept as JSON (name, endpoint, token, filters, retry and worker settings) build their own queue; Close waits for queued and in-flight deliveries before stopping it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: run webhook targets stored in /etc/seaweedfs when notification.webhook.managed is set The targets in /etc/seaweedfs/notification_webhooks.json start with the filer and are rebuilt from the store whenever the file changes on any filer, alongside notification.toml. The previous queue drains before it stops. Events for the file itself, which holds bearer tokens, are not sent to notification queues. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * admin: manage filer webhook targets A Webhooks page under Management lists, adds, edits and deletes the targets kept in /etc/seaweedfs/notification_webhooks.json. Bearer tokens are write-only: the API reports whether one is set, and an empty token on edit keeps the stored one. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: keep the webhook queue when the stored targets did not change Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: keep a replaced webhook queue open until its senders hand over Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * admin: read chunk-backed webhook targets before editing them Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * shell: keep the stored webhook targets out of fs.meta.notify Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * webhook: reject negative managed target settings Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * webhook: no signal handler on queues that reloads replace Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: swap the webhook queue without waiting behind blocked senders The read lock held across SendMessage made a reload wait for any sender blocked on a saturated target, and the waiting writer stalled every other filer write with it, which is when an operator disables that target. Close now refuses new sends and waits for the senders already inside SendMessage, and a send that hits the closing queue retries on its replacement. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * webhook: release blocked test handlers before the server shuts down Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: always run the webhook targets managed in the admin UI Drop the notification.webhook.managed switch: every filer loads and live reloads the stored targets. Without stored targets no queue is created. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * scaffold: place the admin-managed webhook note above enabled Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * scaffold: separate the admin-managed webhook note Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * scaffold: note that admin-managed webhooks follow the notification queue Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: retry loading webhook targets after a failed read Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: retire the admin-managed webhook queue on shutdown Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
476 lines
12 KiB
Go
476 lines
12 KiB
Go
package webhook
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/ThreeDotsLabs/watermill"
|
|
"github.com/ThreeDotsLabs/watermill/message"
|
|
"github.com/ThreeDotsLabs/watermill/message/router/middleware"
|
|
"github.com/ThreeDotsLabs/watermill/message/router/plugin"
|
|
"github.com/ThreeDotsLabs/watermill/pubsub/gochannel"
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/notification"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
"google.golang.org/protobuf/proto"
|
|
)
|
|
|
|
func init() {
|
|
notification.MessageQueues = append(notification.MessageQueues, &Queue{})
|
|
}
|
|
|
|
// target is one webhook endpoint with its own filter, http client and
|
|
// concurrency budget. The flat [notification.webhook] section is the unnamed
|
|
// target; each enabled [notification.webhook.<name>] subsection adds another.
|
|
type target struct {
|
|
name string
|
|
config *config
|
|
filter *filter
|
|
client client
|
|
sem chan struct{}
|
|
inbox chan *message.Message
|
|
pending chan struct{}
|
|
}
|
|
|
|
func newTarget(name string, cfg *config) (*target, error) {
|
|
t := &target{
|
|
name: name,
|
|
config: cfg,
|
|
filter: newFilter(cfg),
|
|
sem: make(chan struct{}, cfg.nWorkers),
|
|
inbox: make(chan *message.Message, cfg.bufferSize),
|
|
pending: make(chan struct{}, cfg.bufferSize),
|
|
}
|
|
hClient, err := newHTTPClient(cfg)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create webhook http client: %w", err)
|
|
}
|
|
t.client = hClient
|
|
return t, nil
|
|
}
|
|
|
|
func (t *target) topic() string {
|
|
if t.name == "" {
|
|
return pubSubTopicName
|
|
}
|
|
return pubSubTopicName + "_" + t.name
|
|
}
|
|
|
|
func (t *target) handlerName() string {
|
|
if t.name == "" {
|
|
return "webhook_handler"
|
|
}
|
|
return "webhook_handler_" + t.name
|
|
}
|
|
|
|
type Queue struct {
|
|
routers []*message.Router
|
|
queueChannel *gochannel.GoChannel
|
|
targets []*target
|
|
|
|
ctx context.Context
|
|
cancel context.CancelFunc
|
|
stopped chan struct{}
|
|
// managed queues are replaced on reload and stopped with Close
|
|
managed bool
|
|
closing atomic.Bool
|
|
senders atomic.Int64
|
|
}
|
|
|
|
// ErrQueueClosed is returned by SendMessage once Close has started.
|
|
var ErrQueueClosed = errors.New("webhook queue closed")
|
|
|
|
func (w *Queue) GetName() string {
|
|
return queueName
|
|
}
|
|
|
|
// HasEnabledSubconfig reports whether any [notification.webhook.<name>]
|
|
// subsection is enabled, so the queue activates on named targets even when the
|
|
// flat [notification.webhook] section is absent.
|
|
func (w *Queue) HasEnabledSubconfig(configuration util.Configuration, prefix string) bool {
|
|
return len(namedTargetPrefixes(configuration, prefix)) > 0
|
|
}
|
|
|
|
// stringMapConfig is implemented by *util.ViperProxy; anything narrower simply
|
|
// gets no named-target discovery.
|
|
type stringMapConfig interface {
|
|
GetStringMap(key string) map[string]interface{}
|
|
}
|
|
|
|
func namedTargetPrefixes(configuration util.Configuration, prefix string) []string {
|
|
mg, ok := configuration.(stringMapConfig)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
var prefixes []string
|
|
for k, v := range mg.GetStringMap(strings.TrimSuffix(prefix, ".")) {
|
|
if _, ok := v.(map[string]interface{}); !ok {
|
|
continue
|
|
}
|
|
p := prefix + k + "."
|
|
if !configuration.GetBool(p + "enabled") {
|
|
continue
|
|
}
|
|
prefixes = append(prefixes, p)
|
|
}
|
|
sort.Strings(prefixes)
|
|
return prefixes
|
|
}
|
|
|
|
func buildTargets(configuration util.Configuration, prefix string) ([]*target, error) {
|
|
var prefixes []string
|
|
if configuration.GetBool(prefix + "enabled") {
|
|
prefixes = append(prefixes, prefix)
|
|
}
|
|
prefixes = append(prefixes, namedTargetPrefixes(configuration, prefix)...)
|
|
|
|
var targets []*target
|
|
for _, p := range prefixes {
|
|
cfg := newConfigWithDefaults(configuration, p)
|
|
if err := cfg.validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
t, err := newTarget(strings.TrimSuffix(strings.TrimPrefix(p, prefix), "."), cfg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
targets = append(targets, t)
|
|
}
|
|
return targets, nil
|
|
}
|
|
|
|
func (w *Queue) SendMessage(key string, msg proto.Message) error {
|
|
w.senders.Add(1)
|
|
defer w.senders.Add(-1)
|
|
if w.closing.Load() {
|
|
return ErrQueueClosed
|
|
}
|
|
eventNotification, ok := msg.(*filer_pb.EventNotification)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
|
|
var deferred []routedMessage
|
|
for _, t := range w.targets {
|
|
if t.filter != nil && !t.filter.shouldPublish(key, eventNotification) {
|
|
continue
|
|
}
|
|
m := newWebhookMessage(key, msg)
|
|
if m == nil {
|
|
continue
|
|
}
|
|
wMsg, err := m.toWaterMillMessage()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
wMsg.Metadata.Set("target", t.name)
|
|
// a free reservation routes the event without waiting on slower
|
|
// targets; a saturated target defers to the blocking pass below so
|
|
// it applies backpressure instead of dropping or starving the rest
|
|
select {
|
|
case t.pending <- struct{}{}:
|
|
if err := t.enqueue(w.ctx, wMsg); err != nil {
|
|
return err
|
|
}
|
|
default:
|
|
deferred = append(deferred, routedMessage{t, wMsg})
|
|
}
|
|
}
|
|
for _, r := range deferred {
|
|
select {
|
|
case r.target.pending <- struct{}{}:
|
|
case <-w.ctx.Done():
|
|
return w.ctx.Err()
|
|
}
|
|
if err := r.target.enqueue(w.ctx, r.msg); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type routedMessage struct {
|
|
target *target
|
|
msg *message.Message
|
|
}
|
|
|
|
func (t *target) enqueue(ctx context.Context, wMsg *message.Message) error {
|
|
select {
|
|
case t.inbox <- wMsg:
|
|
return nil
|
|
case <-ctx.Done():
|
|
t.releasePending()
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (w *webhookMessage) toWaterMillMessage() (*message.Message, error) {
|
|
payload, err := proto.Marshal(w.Notification)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
msg := message.NewMessage(watermill.NewUUID(), payload)
|
|
// Set event type and key as metadata
|
|
msg.Metadata.Set("event_type", w.EventType)
|
|
msg.Metadata.Set("key", w.Key)
|
|
|
|
return msg, nil
|
|
}
|
|
|
|
func (w *Queue) Initialize(configuration util.Configuration, prefix string) error {
|
|
targets, err := buildTargets(configuration, prefix)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(targets) == 0 {
|
|
return fmt.Errorf("webhook endpoint is required")
|
|
}
|
|
|
|
return w.initialize(targets)
|
|
}
|
|
|
|
func (w *Queue) initialize(targets []*target) error {
|
|
w.ctx, w.cancel = context.WithCancel(context.Background())
|
|
w.stopped = make(chan struct{})
|
|
w.targets = targets
|
|
|
|
if err := w.setupWatermillQueue(targets); err != nil {
|
|
return fmt.Errorf("failed to setup watermill queue: %w", err)
|
|
}
|
|
if err := w.logDeadLetterMessages(); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (w *Queue) setupWatermillQueue(targets []*target) error {
|
|
logger := watermill.NewStdLogger(false, false)
|
|
bufferSize := 0
|
|
for _, t := range targets {
|
|
if t.config.bufferSize > bufferSize {
|
|
bufferSize = t.config.bufferSize
|
|
}
|
|
}
|
|
pubSubConfig := gochannel.Config{
|
|
OutputChannelBuffer: int64(bufferSize),
|
|
Persistent: false,
|
|
}
|
|
w.queueChannel = gochannel.NewGoChannel(pubSubConfig, logger)
|
|
|
|
var wg sync.WaitGroup
|
|
for _, t := range targets {
|
|
// One router per target so each keeps its own retry/backoff settings.
|
|
// Handlers subscribe to the target's own topic, so a message is
|
|
// delivered to an endpoint exactly once per matching target.
|
|
router, err := message.NewRouter(
|
|
message.RouterConfig{
|
|
CloseTimeout: 60 * time.Second,
|
|
},
|
|
logger,
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create router: %w", err)
|
|
}
|
|
w.routers = append(w.routers, router)
|
|
|
|
retryMiddleware := middleware.Retry{
|
|
MaxRetries: t.config.maxRetries,
|
|
InitialInterval: time.Duration(t.config.backoffSeconds) * time.Second,
|
|
MaxInterval: time.Duration(t.config.maxBackoffSeconds) * time.Second,
|
|
Multiplier: 2.0,
|
|
RandomizationFactor: 0.3,
|
|
Logger: logger,
|
|
}.Middleware
|
|
|
|
poisonQueue, err := middleware.PoisonQueue(w.queueChannel, deadLetterTopic)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create poison queue: %w", err)
|
|
}
|
|
|
|
if !w.managed {
|
|
router.AddPlugin(plugin.SignalsHandler)
|
|
}
|
|
// outermost: the reservation SendMessage took releases once the whole
|
|
// chain (retries, dead letter) is done, bounding in-flight deliveries.
|
|
// poison wraps retry so the delivery is retried before dead-lettering.
|
|
releasePending := func(h message.HandlerFunc) message.HandlerFunc {
|
|
return func(msg *message.Message) ([]*message.Message, error) {
|
|
defer t.releasePending()
|
|
return h(msg)
|
|
}
|
|
}
|
|
router.AddMiddleware(releasePending, poisonQueue, retryMiddleware)
|
|
|
|
t := t
|
|
router.AddConsumerHandler(
|
|
t.handlerName(),
|
|
t.topic(),
|
|
w.queueChannel,
|
|
t.handleWebhook,
|
|
)
|
|
|
|
routerStopped := make(chan struct{})
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
defer close(routerStopped)
|
|
if err := router.Run(w.ctx); err != nil && !errors.Is(err, context.Canceled) {
|
|
glog.Errorf("webhook pubsub worker stopped with error: %v", err)
|
|
}
|
|
glog.Info("webhook pubsub worker stopped")
|
|
}()
|
|
|
|
go w.forwardTargetMessages(t)
|
|
|
|
// without a subscriber GoChannel drops the publish as undeliverable
|
|
select {
|
|
case <-router.Running():
|
|
case <-routerStopped:
|
|
return fmt.Errorf("webhook router for target %q stopped before it started", t.name)
|
|
case <-w.ctx.Done():
|
|
return w.ctx.Err()
|
|
}
|
|
}
|
|
|
|
// the queue context lives until every target's worker has stopped so the
|
|
// dead letter logger keeps running while any of them is still up
|
|
go func() {
|
|
wg.Wait()
|
|
w.cancel()
|
|
close(w.stopped)
|
|
}()
|
|
|
|
return nil
|
|
}
|
|
|
|
// Close stops the queue once every sender already inside SendMessage has
|
|
// returned and every target has delivered or dead-lettered what it holds, or
|
|
// when drainTimeout passes.
|
|
func (w *Queue) Close(drainTimeout time.Duration) {
|
|
w.closing.Store(true)
|
|
deadline := time.Now().Add(drainTimeout)
|
|
for (w.senders.Load() > 0 || !w.drained()) && time.Now().Before(deadline) {
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
w.cancel()
|
|
<-w.stopped
|
|
if err := w.queueChannel.Close(); err != nil {
|
|
glog.Warningf("close webhook queue: %v", err)
|
|
}
|
|
}
|
|
|
|
func (w *Queue) drained() bool {
|
|
for _, t := range w.targets {
|
|
if len(t.pending) > 0 {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// forwardTargetMessages publishes each queued event to the target's topic in
|
|
// order. Publish detaches its delivery, so the pending reservation is what
|
|
// keeps a saturated target from queuing without bound.
|
|
func (w *Queue) forwardTargetMessages(t *target) {
|
|
for {
|
|
select {
|
|
case msg := <-t.inbox:
|
|
if err := w.queueChannel.Publish(t.topic(), msg); err != nil {
|
|
glog.Errorf("webhook target %q failed to publish message: %v", t.name, err)
|
|
t.releasePending()
|
|
}
|
|
case <-w.ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// releasePending frees one delivery reservation. It never blocks: a nacked
|
|
// message GoChannel resends reaches the release again after its token is gone.
|
|
func (t *target) releasePending() {
|
|
select {
|
|
case <-t.pending:
|
|
default:
|
|
}
|
|
}
|
|
|
|
func (t *target) handleWebhook(msg *message.Message) error {
|
|
// Acquire semaphore slot (blocks if at capacity)
|
|
t.sem <- struct{}{}
|
|
defer func() { <-t.sem }()
|
|
|
|
var n filer_pb.EventNotification
|
|
if err := proto.Unmarshal(msg.Payload, &n); err != nil {
|
|
glog.Errorf("failed to unmarshal protobuf message: %v", err)
|
|
return err
|
|
}
|
|
|
|
// Reconstruct webhook message from metadata and payload
|
|
webhookMsg := &webhookMessage{
|
|
Key: msg.Metadata.Get("key"),
|
|
EventType: msg.Metadata.Get("event_type"),
|
|
Notification: &n,
|
|
}
|
|
|
|
if err := t.client.sendMessage(webhookMsg); err != nil {
|
|
glog.Errorf("webhook target %q failed to send message for %s: %v", t.name, webhookMsg.Key, err)
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (w *Queue) logDeadLetterMessages() error {
|
|
ch, err := w.queueChannel.Subscribe(w.ctx, deadLetterTopic)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
go func() {
|
|
for {
|
|
select {
|
|
case msg, ok := <-ch:
|
|
if !ok {
|
|
glog.Info("dead letter channel closed")
|
|
return
|
|
}
|
|
if msg == nil {
|
|
glog.Errorf("received nil message from dead letter channel")
|
|
continue
|
|
}
|
|
key := "unknown"
|
|
targetName := ""
|
|
if msg.Metadata != nil {
|
|
if keyValue, exists := msg.Metadata["key"]; exists {
|
|
key = keyValue
|
|
}
|
|
targetName = msg.Metadata["target"]
|
|
}
|
|
payload := ""
|
|
if msg.Payload != nil {
|
|
var n filer_pb.EventNotification
|
|
if err := proto.Unmarshal(msg.Payload, &n); err != nil {
|
|
payload = fmt.Sprintf("failed to unmarshal payload: %v", err)
|
|
} else {
|
|
payload = n.String()
|
|
}
|
|
}
|
|
glog.Errorf("received dead letter message: %s, key: %s, target: %s", payload, key, targetName)
|
|
case <-w.ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
|
|
return nil
|
|
}
|