admin: manage filer webhook targets from the UI with live reload (#12690)

* webhook: share the default target settings

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* webhook: managed webhook targets with a drainable queue

Targets kept as JSON (name, endpoint, token, filters, retry and worker
settings) build their own queue; Close waits for queued and in-flight
deliveries before stopping it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: run webhook targets stored in /etc/seaweedfs when notification.webhook.managed is set

The targets in /etc/seaweedfs/notification_webhooks.json start with the
filer and are rebuilt from the store whenever the file changes on any
filer, alongside notification.toml. The previous queue drains before it
stops. Events for the file itself, which holds bearer tokens, are not
sent to notification queues.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* admin: manage filer webhook targets

A Webhooks page under Management lists, adds, edits and deletes the
targets kept in /etc/seaweedfs/notification_webhooks.json. Bearer
tokens are write-only: the API reports whether one is set, and an empty
token on edit keeps the stored one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: keep the webhook queue when the stored targets did not change

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: keep a replaced webhook queue open until its senders hand over

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* admin: read chunk-backed webhook targets before editing them

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* shell: keep the stored webhook targets out of fs.meta.notify

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* webhook: reject negative managed target settings

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* webhook: no signal handler on queues that reloads replace

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: swap the webhook queue without waiting behind blocked senders

The read lock held across SendMessage made a reload wait for any sender
blocked on a saturated target, and the waiting writer stalled every
other filer write with it, which is when an operator disables that
target. Close now refuses new sends and waits for the senders already
inside SendMessage, and a send that hits the closing queue retries on
its replacement.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* webhook: release blocked test handlers before the server shuts down

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: always run the webhook targets managed in the admin UI

Drop the notification.webhook.managed switch: every filer loads and live
reloads the stored targets. Without stored targets no queue is created.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* scaffold: place the admin-managed webhook note above enabled

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* scaffold: separate the admin-managed webhook note

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* scaffold: note that admin-managed webhooks follow the notification queue

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: retry loading webhook targets after a failed read

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: retire the admin-managed webhook queue on shutdown

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed 2026-11-17 16:54:54 -08:00
1 parent 386385101d
commit 95f46c8fff
19 files changed
+1844 -250

No files matched your search

+150
View File
@@ -0,0 +1,150 @@
package webhook
import (
"encoding/json"
"fmt"
"net/url"
"regexp"
"strings"
)
// ManagedTarget is a webhook target kept in the filer and edited from the
// admin UI. Zero numeric settings take the same defaults as notification.toml.
type ManagedTarget struct {
Name string `json:"name"`
Enabled bool `json:"enabled"`
Endpoint string `json:"endpoint"`
BearerToken string `json:"bearer_token,omitempty"`
EventTypes []string `json:"event_types,omitempty"`
PathPrefixes []string `json:"path_prefixes,omitempty"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
MaxRetries int `json:"max_retries,omitempty"`
BackoffSeconds int `json:"backoff_seconds,omitempty"`
MaxBackoffSeconds int `json:"max_backoff_seconds,omitempty"`
Workers int `json:"workers,omitempty"`
BufferSize int `json:"buffer_size,omitempty"`
}
type ManagedConfig struct {
Targets []ManagedTarget `json:"targets"`
}
var managedTargetName = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`)
func ParseManagedConfig(data []byte) (*ManagedConfig, error) {
c := &ManagedConfig{}
if len(strings.TrimSpace(string(data))) == 0 {
return c, nil
}
if err := json.Unmarshal(data, c); err != nil {
return nil, fmt.Errorf("parse webhook targets: %w", err)
}
if err := c.Validate(); err != nil {
return nil, err
}
return c, nil
}
func (c *ManagedConfig) Validate() error {
seen := make(map[string]bool, len(c.Targets))
for i := range c.Targets {
t := &c.Targets[i]
if seen[t.Name] {
return fmt.Errorf("duplicate webhook target %q", t.Name)
}
seen[t.Name] = true
if err := t.Validate(); err != nil {
return err
}
}
return nil
}
func (t *ManagedTarget) Validate() error {
if !managedTargetName.MatchString(t.Name) {
return fmt.Errorf("webhook target name %q must be 1-64 letters, digits, '-' or '_'", t.Name)
}
u, err := url.Parse(t.Endpoint)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return fmt.Errorf("webhook target %q: endpoint must be an http or https URL", t.Name)
}
for _, et := range t.EventTypes {
if !eventType(et).valid() {
return fmt.Errorf("webhook target %q: invalid event type %q", t.Name, et)
}
}
for _, p := range t.PathPrefixes {
if !strings.HasPrefix(p, "/") {
return fmt.Errorf("webhook target %q: path prefix %q must start with '/'", t.Name, p)
}
}
for field, v := range t.settings() {
if v < 0 {
return fmt.Errorf("webhook target %q: %s must not be negative", t.Name, field)
}
}
if err := t.config().validate(); err != nil {
return fmt.Errorf("webhook target %q: %w", t.Name, err)
}
return nil
}
func (t *ManagedTarget) settings() map[string]int {
return map[string]int{
"timeout_seconds": t.TimeoutSeconds,
"max_retries": t.MaxRetries,
"backoff_seconds": t.BackoffSeconds,
"max_backoff_seconds": t.MaxBackoffSeconds,
"workers": t.Workers,
"buffer_size": t.BufferSize,
}
}
func (t *ManagedTarget) config() *config {
c := defaultConfig()
c.endpoint = t.Endpoint
c.authBearerToken = t.BearerToken
c.eventTypes = t.EventTypes
c.pathPrefixes = t.PathPrefixes
for dst, v := range map[*int]int{
&c.timeoutSeconds: t.TimeoutSeconds,
&c.maxRetries: t.MaxRetries,
&c.backoffSeconds: t.BackoffSeconds,
&c.maxBackoffSeconds: t.MaxBackoffSeconds,
&c.nWorkers: t.Workers,
&c.bufferSize: t.BufferSize,
} {
if v > 0 {
*dst = v
}
}
return c
}
// NewManagedQueue starts delivering to the enabled targets; it returns nil
// when none is enabled.
func NewManagedQueue(c *ManagedConfig) (*Queue, error) {
if err := c.Validate(); err != nil {
return nil, err
}
var targets []*target
for i := range c.Targets {
if !c.Targets[i].Enabled {
continue
}
t, err := newTarget(c.Targets[i].Name, c.Targets[i].config())
if err != nil {
return nil, err
}
targets = append(targets, t)
}
if len(targets) == 0 {
return nil, nil
}
w := &Queue{managed: true}
if err := w.initialize(targets); err != nil {
w.cancel()
return nil, err
}
return w, nil
}
+165
View File
@@ -0,0 +1,165 @@
package webhook
import (
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func TestParseManagedConfig(t *testing.T) {
c, err := ParseManagedConfig(nil)
if err != nil || len(c.Targets) != 0 {
t.Fatalf("empty content: got %+v, %v", c, err)
}
c, err = ParseManagedConfig([]byte(`{"targets":[{"name":"ingest","enabled":true,"endpoint":"https://example.com/hook","event_types":["create"],"path_prefixes":["/buckets/a"],"max_retries":5}]}`))
if err != nil {
t.Fatalf("ParseManagedConfig: %v", err)
}
cfg := c.Targets[0].config()
if cfg.maxRetries != 5 || cfg.timeoutSeconds != 10 || cfg.nWorkers != 5 || cfg.bufferSize != 10_000 {
t.Fatalf("unexpected settings %+v", cfg)
}
for name, body := range map[string]string{
"bad name": `{"targets":[{"name":"a b","endpoint":"https://example.com"}]}`,
"duplicate": `{"targets":[{"name":"a","endpoint":"https://example.com"},{"name":"a","endpoint":"https://example.com"}]}`,
"no scheme": `{"targets":[{"name":"a","endpoint":"example.com/hook"}]}`,
"ftp endpoint": `{"targets":[{"name":"a","endpoint":"ftp://example.com/hook"}]}`,
"event type": `{"targets":[{"name":"a","endpoint":"https://example.com","event_types":["chmod"]}]}`,
"relative path": `{"targets":[{"name":"a","endpoint":"https://example.com","path_prefixes":["buckets"]}]}`,
"timeout range": `{"targets":[{"name":"a","endpoint":"https://example.com","timeout_seconds":301}]}`,
"neg timeout": `{"targets":[{"name":"a","endpoint":"https://example.com","timeout_seconds":-1}]}`,
"neg workers": `{"targets":[{"name":"a","endpoint":"https://example.com","workers":-1}]}`,
"not json": `targets`,
} {
if _, err := ParseManagedConfig([]byte(body)); err == nil {
t.Errorf("%s: expected an error", name)
}
}
}
func TestManagedQueueDeliversAndDrainsOnClose(t *testing.T) {
var hits atomic.Int32
var auth atomic.Value
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
auth.Store(r.Header.Get("Authorization"))
<-release
hits.Add(1)
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
unblock := sync.OnceFunc(func() { close(release) })
defer unblock()
q, err := NewManagedQueue(&ManagedConfig{Targets: []ManagedTarget{
{Name: "on", Enabled: true, Endpoint: server.URL, BearerToken: "secret", PathPrefixes: []string{"/data/"}},
{Name: "off", Endpoint: "https://example.com/never"},
}})
if err != nil {
t.Fatalf("NewManagedQueue: %v", err)
}
if len(q.targets) != 1 {
t.Fatalf("expected only the enabled target, got %d", len(q.targets))
}
msg := &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "f.txt"}}
for _, key := range []string{"/data/f.txt", "/other/f.txt"} {
if err := q.SendMessage(key, msg); err != nil {
t.Fatalf("SendMessage(%q): %v", key, err)
}
}
closed := make(chan struct{})
go func() {
q.Close(10 * time.Second)
close(closed)
}()
select {
case <-closed:
t.Fatal("Close returned before the in-flight delivery finished")
case <-time.After(300 * time.Millisecond):
}
unblock()
select {
case <-closed:
case <-time.After(10 * time.Second):
t.Fatal("Close did not return after the delivery finished")
}
if got := hits.Load(); got != 1 {
t.Fatalf("expected 1 delivery, got %d", got)
}
if got, _ := auth.Load().(string); !strings.HasSuffix(got, "secret") {
t.Fatalf("expected the bearer token, got %q", got)
}
}
func TestManagedQueueNoEnabledTargets(t *testing.T) {
q, err := NewManagedQueue(&ManagedConfig{Targets: []ManagedTarget{{Name: "off", Endpoint: "https://example.com"}}})
if err != nil || q != nil {
t.Fatalf("expected no queue, got %v, %v", q, err)
}
}
func TestManagedQueueCloseWaitsForBlockedSender(t *testing.T) {
var hits atomic.Int32
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
<-release
hits.Add(1)
}))
defer server.Close()
unblock := sync.OnceFunc(func() { close(release) })
defer unblock()
q, err := NewManagedQueue(&ManagedConfig{Targets: []ManagedTarget{
{Name: "slow", Enabled: true, Endpoint: server.URL, Workers: 1, BufferSize: 100},
}})
if err != nil {
t.Fatalf("NewManagedQueue: %v", err)
}
msg := &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "f.txt"}}
for i := 0; i < 100; i++ {
if err := q.SendMessage("/a", msg); err != nil {
t.Fatalf("SendMessage: %v", err)
}
}
blocked := make(chan error, 1)
go func() { blocked <- q.SendMessage("/b", msg) }()
for q.senders.Load() == 0 {
time.Sleep(time.Millisecond)
}
closed := make(chan struct{})
go func() {
q.Close(10 * time.Second)
close(closed)
}()
for !q.closing.Load() {
time.Sleep(time.Millisecond)
}
if err := q.SendMessage("/c", msg); err != ErrQueueClosed {
t.Fatalf("expected ErrQueueClosed after Close started, got %v", err)
}
unblock()
if err := <-blocked; err != nil {
t.Fatalf("blocked sender: %v", err)
}
select {
case <-closed:
case <-time.After(10 * time.Second):
t.Fatal("Close did not return")
}
if got := hits.Load(); got != 101 {
t.Fatalf("expected both accepted events delivered, got %d", got)
}
}
+8 -4
View File
@@ -69,10 +69,8 @@ type config struct {
pathPrefixes []string
}
func newConfigWithDefaults(configuration util.Configuration, prefix string) *config {
c := &config{
endpoint: configuration.GetString(prefix + "endpoint"),
authBearerToken: configuration.GetString(prefix + "bearer_token"),
func defaultConfig() *config {
return &config{
timeoutSeconds: 10,
maxRetries: 3,
backoffSeconds: 3,
@@ -80,6 +78,12 @@ func newConfigWithDefaults(configuration util.Configuration, prefix string) *con
nWorkers: 5,
bufferSize: 10_000,
}
}
func newConfigWithDefaults(configuration util.Configuration, prefix string) *config {
c := defaultConfig()
c.endpoint = configuration.GetString(prefix + "endpoint")
c.authBearerToken = configuration.GetString(prefix + "bearer_token")
if bufferSize := configuration.GetInt(prefix + "buffer_size"); bufferSize > 0 {
c.bufferSize = bufferSize
+49 -6
View File
@@ -7,6 +7,7 @@ import (
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/ThreeDotsLabs/watermill"
@@ -40,9 +41,9 @@ type target struct {
func newTarget(name string, cfg *config) (*target, error) {
t := &target{
name: name,
config: cfg,
filter: newFilter(cfg),
name: name,
config: cfg,
filter: newFilter(cfg),
sem: make(chan struct{}, cfg.nWorkers),
inbox: make(chan *message.Message, cfg.bufferSize),
pending: make(chan struct{}, cfg.bufferSize),
@@ -74,10 +75,18 @@ type Queue struct {
queueChannel *gochannel.GoChannel
targets []*target
ctx context.Context
cancel context.CancelFunc
ctx context.Context
cancel context.CancelFunc
stopped chan struct{}
// managed queues are replaced on reload and stopped with Close
managed bool
closing atomic.Bool
senders atomic.Int64
}
// ErrQueueClosed is returned by SendMessage once Close has started.
var ErrQueueClosed = errors.New("webhook queue closed")
func (w *Queue) GetName() string {
return queueName
}
@@ -138,6 +147,11 @@ func buildTargets(configuration util.Configuration, prefix string) ([]*target, e
}
func (w *Queue) SendMessage(key string, msg proto.Message) error {
w.senders.Add(1)
defer w.senders.Add(-1)
if w.closing.Load() {
return ErrQueueClosed
}
eventNotification, ok := msg.(*filer_pb.EventNotification)
if !ok {
return nil
@@ -225,6 +239,7 @@ func (w *Queue) Initialize(configuration util.Configuration, prefix string) erro
func (w *Queue) initialize(targets []*target) error {
w.ctx, w.cancel = context.WithCancel(context.Background())
w.stopped = make(chan struct{})
w.targets = targets
if err := w.setupWatermillQueue(targets); err != nil {
@@ -281,7 +296,9 @@ func (w *Queue) setupWatermillQueue(targets []*target) error {
return fmt.Errorf("failed to create poison queue: %w", err)
}
router.AddPlugin(plugin.SignalsHandler)
if !w.managed {
router.AddPlugin(plugin.SignalsHandler)
}
// outermost: the reservation SendMessage took releases once the whole
// chain (retries, dead letter) is done, bounding in-flight deliveries.
// poison wraps retry so the delivery is retried before dead-lettering.
@@ -329,11 +346,37 @@ func (w *Queue) setupWatermillQueue(targets []*target) error {
go func() {
wg.Wait()
w.cancel()
close(w.stopped)
}()
return nil
}
// Close stops the queue once every sender already inside SendMessage has
// returned and every target has delivered or dead-lettered what it holds, or
// when drainTimeout passes.
func (w *Queue) Close(drainTimeout time.Duration) {
w.closing.Store(true)
deadline := time.Now().Add(drainTimeout)
for (w.senders.Load() > 0 || !w.drained()) && time.Now().Before(deadline) {
time.Sleep(50 * time.Millisecond)
}
w.cancel()
<-w.stopped
if err := w.queueChannel.Close(); err != nil {
glog.Warningf("close webhook queue: %v", err)
}
}
func (w *Queue) drained() bool {
for _, t := range w.targets {
if len(t.pending) > 0 {
return false
}
}
return true
}
// forwardTargetMessages publishes each queued event to the target's topic in
// order. Publish detaches its delivery, so the pending reservation is what
// keeps a saturated target from queuing without bound.