From 95f46c8fff95917f2df88a88f64a0046fdf863bd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 17 Nov 2026 16:54:54 -0800 Subject: [PATCH] admin: manage filer webhook targets from the UI with live reload (#12690) * webhook: share the default target settings Co-Authored-By: Claude Opus 5.5 * webhook: managed webhook targets with a drainable queue Targets kept as JSON (name, endpoint, token, filters, retry and worker settings) build their own queue; Close waits for queued and in-flight deliveries before stopping it. Co-Authored-By: Claude Opus 5.5 * filer: run webhook targets stored in /etc/seaweedfs when notification.webhook.managed is set The targets in /etc/seaweedfs/notification_webhooks.json start with the filer and are rebuilt from the store whenever the file changes on any filer, alongside notification.toml. The previous queue drains before it stops. Events for the file itself, which holds bearer tokens, are not sent to notification queues. Co-Authored-By: Claude Opus 5.5 * admin: manage filer webhook targets A Webhooks page under Management lists, adds, edits and deletes the targets kept in /etc/seaweedfs/notification_webhooks.json. Bearer tokens are write-only: the API reports whether one is set, and an empty token on edit keeps the stored one. Co-Authored-By: Claude Opus 5.5 * filer: keep the webhook queue when the stored targets did not change Co-Authored-By: Claude Opus 5.5 * filer: keep a replaced webhook queue open until its senders hand over Co-Authored-By: Claude Opus 5.5 * admin: read chunk-backed webhook targets before editing them Co-Authored-By: Claude Opus 5.5 * shell: keep the stored webhook targets out of fs.meta.notify Co-Authored-By: Claude Opus 5.5 * webhook: reject negative managed target settings Co-Authored-By: Claude Opus 5.5 * webhook: no signal handler on queues that reloads replace Co-Authored-By: Claude Opus 5.5 * filer: swap the webhook queue without waiting behind blocked senders The read lock held across SendMessage made a reload wait for any sender blocked on a saturated target, and the waiting writer stalled every other filer write with it, which is when an operator disables that target. Close now refuses new sends and waits for the senders already inside SendMessage, and a send that hits the closing queue retries on its replacement. Co-Authored-By: Claude Opus 5.5 * webhook: release blocked test handlers before the server shuts down Co-Authored-By: Claude Opus 5.5 * filer: always run the webhook targets managed in the admin UI Drop the notification.webhook.managed switch: every filer loads and live reloads the stored targets. Without stored targets no queue is created. Co-Authored-By: Claude Opus 5.5 * scaffold: place the admin-managed webhook note above enabled Co-Authored-By: Claude Opus 5.5 * scaffold: separate the admin-managed webhook note Co-Authored-By: Claude Opus 5.5 * scaffold: note that admin-managed webhooks follow the notification queue Co-Authored-By: Claude Opus 5.5 * filer: retry loading webhook targets after a failed read Co-Authored-By: Claude Opus 5.5 * filer: retire the admin-managed webhook queue on shutdown Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- weed/admin/dash/notification_webhooks.go | 192 +++++++ weed/admin/handlers/admin_handlers.go | 25 + .../handlers/admin_handlers_routes_test.go | 14 + .../view/app/notification_webhooks.templ | 283 ++++++++++ .../view/app/notification_webhooks_templ.go | 362 +++++++++++++ weed/admin/view/layout/layout.templ | 5 + weed/admin/view/layout/layout_templ.go | 483 +++++++++--------- weed/command/scaffold/notification.toml | 6 + weed/filer/filer.go | 1 + weed/filer/filer_notification_webhooks.go | 146 ++++++ .../filer/filer_notification_webhooks_test.go | 176 +++++++ weed/filer/filer_notify.go | 13 +- weed/filer/filer_on_meta_event.go | 1 + weed/notification/webhook/managed.go | 150 ++++++ weed/notification/webhook/managed_test.go | 165 ++++++ weed/notification/webhook/types.go | 12 +- weed/notification/webhook/webhook_queue.go | 55 +- weed/server/filer_server.go | 1 + weed/shell/command_fs_meta_notify.go | 4 + 19 files changed, 1844 insertions(+), 250 deletions(-) create mode 100644 weed/admin/dash/notification_webhooks.go create mode 100644 weed/admin/view/app/notification_webhooks.templ create mode 100644 weed/admin/view/app/notification_webhooks_templ.go create mode 100644 weed/filer/filer_notification_webhooks.go create mode 100644 weed/filer/filer_notification_webhooks_test.go create mode 100644 weed/notification/webhook/managed.go create mode 100644 weed/notification/webhook/managed_test.go diff --git a/weed/admin/dash/notification_webhooks.go b/weed/admin/dash/notification_webhooks.go new file mode 100644 index 0000000000..8796410a55 --- /dev/null +++ b/weed/admin/dash/notification_webhooks.go @@ -0,0 +1,192 @@ +package dash + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "sync" + + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/notification/webhook" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" +) + +// NotificationWebhook is a stored webhook target as shown to the browser: +// the bearer token never leaves the server. +type NotificationWebhook struct { + webhook.ManagedTarget + HasBearerToken bool `json:"has_bearer_token"` +} + +type NotificationWebhooksData struct { + Webhooks []NotificationWebhook `json:"webhooks"` + ConfigPath string `json:"config_path"` +} + +// NotificationWebhookRequest creates or replaces a target. An empty +// bearer_token keeps the stored one unless clear_bearer_token is set. +type NotificationWebhookRequest struct { + webhook.ManagedTarget + ClearBearerToken bool `json:"clear_bearer_token"` +} + +var notificationWebhooksMu sync.Mutex + +func (s *AdminServer) readNotificationWebhooks(ctx context.Context) (*webhook.ManagedConfig, error) { + var conf *webhook.ManagedConfig + err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + var buf bytes.Buffer + err := filer.ReadEntry(s.masterClient, client, filer.DirectoryEtcSeaweedFS, filer.NotificationWebhooksConfName, &buf) + if err != nil && !errors.Is(err, filer_pb.ErrNotFound) { + return err + } + conf, err = webhook.ParseManagedConfig(buf.Bytes()) + return err + }) + return conf, err +} + +func (s *AdminServer) saveNotificationWebhooks(ctx context.Context, conf *webhook.ManagedConfig) error { + if err := conf.Validate(); err != nil { + return err + } + content, err := json.MarshalIndent(conf, "", " ") + if err != nil { + return err + } + return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + return filer.SaveInsideFiler(ctx, client, filer.DirectoryEtcSeaweedFS, filer.NotificationWebhooksConfName, content) + }) +} + +func (s *AdminServer) GetNotificationWebhooks(ctx context.Context) (*NotificationWebhooksData, error) { + conf, err := s.readNotificationWebhooks(ctx) + if err != nil { + return nil, err + } + data := &NotificationWebhooksData{ + Webhooks: make([]NotificationWebhook, 0, len(conf.Targets)), + ConfigPath: filer.NotificationWebhooksConfPath, + } + for _, t := range conf.Targets { + hasToken := t.BearerToken != "" + t.BearerToken = "" + data.Webhooks = append(data.Webhooks, NotificationWebhook{ManagedTarget: t, HasBearerToken: hasToken}) + } + return data, nil +} + +func (s *AdminServer) GetNotificationWebhooksAPI(w http.ResponseWriter, r *http.Request) { + data, err := s.GetNotificationWebhooks(r.Context()) + if err != nil { + writeJSONError(w, http.StatusInternalServerError, "Failed to read webhook targets: "+err.Error()) + return + } + writeJSON(w, http.StatusOK, data) +} + +func (s *AdminServer) CreateNotificationWebhookAPI(w http.ResponseWriter, r *http.Request) { + s.putNotificationWebhook(w, r, "", true) +} + +func (s *AdminServer) UpdateNotificationWebhookAPI(w http.ResponseWriter, r *http.Request) { + s.putNotificationWebhook(w, r, mux.Vars(r)["name"], false) +} + +func (s *AdminServer) putNotificationWebhook(w http.ResponseWriter, r *http.Request, name string, create bool) { + if !requireSessionCSRFToken(w, r) { + return + } + var req NotificationWebhookRequest + if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil { + writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error()) + return + } + if create { + name = req.Name + } else if req.Name != "" && req.Name != name { + writeJSONError(w, http.StatusBadRequest, "Webhook target name cannot be changed") + return + } + target := req.ManagedTarget + target.Name = name + + notificationWebhooksMu.Lock() + defer notificationWebhooksMu.Unlock() + conf, err := s.readNotificationWebhooks(r.Context()) + if err != nil { + writeJSONError(w, http.StatusInternalServerError, "Failed to read webhook targets: "+err.Error()) + return + } + index := -1 + for i := range conf.Targets { + if conf.Targets[i].Name == name { + index = i + } + } + switch { + case create && index >= 0: + writeJSONError(w, http.StatusConflict, "Webhook target already exists") + return + case !create && index < 0: + writeJSONError(w, http.StatusNotFound, "Webhook target not found") + return + } + if index >= 0 && target.BearerToken == "" && !req.ClearBearerToken { + target.BearerToken = conf.Targets[index].BearerToken + } + if err := target.Validate(); err != nil { + writeJSONError(w, http.StatusBadRequest, err.Error()) + return + } + if index >= 0 { + conf.Targets[index] = target + } else { + conf.Targets = append(conf.Targets, target) + } + if err := s.saveNotificationWebhooks(r.Context(), conf); err != nil { + writeJSONError(w, http.StatusInternalServerError, "Failed to save webhook targets: "+err.Error()) + return + } + status := http.StatusOK + if create { + status = http.StatusCreated + } + hasToken := target.BearerToken != "" + target.BearerToken = "" + writeJSON(w, status, NotificationWebhook{ManagedTarget: target, HasBearerToken: hasToken}) +} + +func (s *AdminServer) DeleteNotificationWebhookAPI(w http.ResponseWriter, r *http.Request) { + if !requireSessionCSRFToken(w, r) { + return + } + name := mux.Vars(r)["name"] + + notificationWebhooksMu.Lock() + defer notificationWebhooksMu.Unlock() + conf, err := s.readNotificationWebhooks(r.Context()) + if err != nil { + writeJSONError(w, http.StatusInternalServerError, "Failed to read webhook targets: "+err.Error()) + return + } + kept := conf.Targets[:0] + for _, t := range conf.Targets { + if t.Name != name { + kept = append(kept, t) + } + } + if len(kept) == len(conf.Targets) { + writeJSONError(w, http.StatusNotFound, "Webhook target not found") + return + } + conf.Targets = kept + if err := s.saveNotificationWebhooks(r.Context(), conf); err != nil { + writeJSONError(w, http.StatusInternalServerError, "Failed to save webhook targets: "+err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]string{"message": "Webhook target deleted"}) +} diff --git a/weed/admin/handlers/admin_handlers.go b/weed/admin/handlers/admin_handlers.go index a9807185c1..b3a33cd638 100644 --- a/weed/admin/handlers/admin_handlers.go +++ b/weed/admin/handlers/admin_handlers.go @@ -126,6 +126,7 @@ func (h *AdminHandlers) registerUIRoutes(r *mux.Router) { // File browser routes r.HandleFunc("/files", h.fileBrowserHandlers.ShowFileBrowser).Methods(http.MethodGet) + r.HandleFunc("/notifications/webhooks", h.ShowNotificationWebhooks).Methods(http.MethodGet) // Cluster management routes r.HandleFunc("/cluster/masters", h.clusterHandlers.ShowClusterMasters).Methods(http.MethodGet) @@ -275,6 +276,12 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) { filesApi.HandleFunc("/metadata", h.fileBrowserHandlers.ExportMetadata).Methods(http.MethodGet) filesApi.HandleFunc("/list-folders", h.fileBrowserHandlers.ListFolders).Methods(http.MethodGet) + webhooksApi := api.PathPrefix("/notifications/webhooks").Subrouter() + webhooksApi.HandleFunc("", h.adminServer.GetNotificationWebhooksAPI).Methods(http.MethodGet) + webhooksApi.Handle("", wrapWrite(h.adminServer.CreateNotificationWebhookAPI)).Methods(http.MethodPost) + webhooksApi.Handle("/{name}", wrapWrite(h.adminServer.UpdateNotificationWebhookAPI)).Methods(http.MethodPut) + webhooksApi.Handle("/{name}", wrapWrite(h.adminServer.DeleteNotificationWebhookAPI)).Methods(http.MethodDelete) + volumeApi := api.PathPrefix("/volumes").Subrouter() volumeApi.HandleFunc("/export", h.clusterHandlers.ExportClusterVolumes).Methods(http.MethodGet) volumeApi.Handle("/{id}/{server}/vacuum", wrapWrite(h.clusterHandlers.VacuumVolume)).Methods(http.MethodPost) @@ -386,6 +393,24 @@ func (h *AdminHandlers) ShowS3TablesBuckets(w http.ResponseWriter, r *http.Reque } } +// ShowNotificationWebhooks renders the filer webhook targets page +func (h *AdminHandlers) ShowNotificationWebhooks(w http.ResponseWriter, r *http.Request) { + username := h.getUsername(r) + data, err := h.adminServer.GetNotificationWebhooks(r.Context()) + if err != nil { + writeJSONError(w, http.StatusInternalServerError, "Failed to get webhook targets: "+err.Error()) + return + } + + w.Header().Set("Content-Type", "text/html") + component := app.NotificationWebhooks(*data) + viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context())) + layoutComponent := layout.Layout(viewCtx, component) + if err := layoutComponent.Render(r.Context(), w); err != nil { + writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error()) + } +} + // ShowS3VectorsBuckets renders the vector buckets page func (h *AdminHandlers) ShowS3VectorsBuckets(w http.ResponseWriter, r *http.Request) { username := h.getUsername(r) diff --git a/weed/admin/handlers/admin_handlers_routes_test.go b/weed/admin/handlers/admin_handlers_routes_test.go index 1cc0728cad..96c133f22e 100644 --- a/weed/admin/handlers/admin_handlers_routes_test.go +++ b/weed/admin/handlers/admin_handlers_routes_test.go @@ -62,6 +62,20 @@ func TestSetupRoutes_RegistersBucketLifecycleAPI_WithAuth(t *testing.T) { assertHasRoute(t, router, http.MethodDelete, "/api/s3/buckets/example/lifecycle") } +func TestSetupRoutes_RegistersNotificationWebhooks(t *testing.T) { + for _, authRequired := range []bool{false, true} { + router := mux.NewRouter() + + newRouteTestAdminHandlers().SetupRoutes(router, authRequired, "admin", "password", "", "", true) + + assertHasRoute(t, router, http.MethodGet, "/notifications/webhooks") + assertHasRoute(t, router, http.MethodGet, "/api/notifications/webhooks") + assertHasRoute(t, router, http.MethodPost, "/api/notifications/webhooks") + assertHasRoute(t, router, http.MethodPut, "/api/notifications/webhooks/example") + assertHasRoute(t, router, http.MethodDelete, "/api/notifications/webhooks/example") + } +} + func TestSetupRoutes_RegistersBucketPolicyAPI_NoAuth(t *testing.T) { router := mux.NewRouter() diff --git a/weed/admin/view/app/notification_webhooks.templ b/weed/admin/view/app/notification_webhooks.templ new file mode 100644 index 0000000000..9460de5346 --- /dev/null +++ b/weed/admin/view/app/notification_webhooks.templ @@ -0,0 +1,283 @@ +package app + +import ( + "strings" + "github.com/seaweedfs/seaweedfs/weed/admin/dash" +) + +templ NotificationWebhooks(data dash.NotificationWebhooksData) { +
+
+
+

+ Webhooks +

+

Send filer events to HTTP endpoints, filtered by path and event type

+
+ +
+ +
+ + Targets are stored in the filer at { data.ConfigPath } and apply to all filers without a restart. +
+ +
+
+ if len(data.Webhooks) == 0 { +
+ +

No webhook targets yet.

+
+ } else { +
+ + + + + + + + + + + + + + for _, hook := range data.Webhooks { + + + + + + + + + + } + +
NameStatusEndpointEventsPath PrefixesTokenActions
{ hook.Name } + if hook.Enabled { + Enabled + } else { + Disabled + } + { hook.Endpoint }{ webhookFilterLabel(hook.EventTypes) }{ webhookFilterLabel(hook.PathPrefixes) } + if hook.HasBearerToken { + + } else { + - + } + +
+ + +
+
+
+ } +
+
+
+ + + + +} + +templ webhookNumberInput(id string, label string, placeholder string) { +
+ + +
+} + +func webhookFilterLabel(values []string) string { + if len(values) == 0 { + return "all" + } + return strings.Join(values, ", ") +} diff --git a/weed/admin/view/app/notification_webhooks_templ.go b/weed/admin/view/app/notification_webhooks_templ.go new file mode 100644 index 0000000000..d1513bcb82 --- /dev/null +++ b/weed/admin/view/app/notification_webhooks_templ.go @@ -0,0 +1,362 @@ +// Code generated by templ - DO NOT EDIT. + +// templ: version: v0.3.1100 + +package app + +//lint:file-ignore SA4006 This context is only used if a nested component is present. + +import "github.com/a-h/templ" +import templruntime "github.com/a-h/templ/runtime" + +import ( + "github.com/seaweedfs/seaweedfs/weed/admin/dash" + "strings" +) + +func NotificationWebhooks(data dash.NotificationWebhooksData) templ.Component { + return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { + templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context + if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { + return templ_7745c5c3_CtxErr + } + templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) + if !templ_7745c5c3_IsBuffer { + defer func() { + templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) + if templ_7745c5c3_Err == nil { + templ_7745c5c3_Err = templ_7745c5c3_BufErr + } + }() + } + ctx = templ.InitializeContext(ctx) + ctx = templ.ClearChildren(ctx) + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

Webhooks

Send filer events to HTTP endpoints, filtered by path and event type

Targets are stored in the filer at ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var2 string + templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(data.ConfigPath) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 24, Col: 70} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, " and apply to all filers without a restart.
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if len(data.Webhooks) == 0 { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "

No webhook targets yet.

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + for _, hook := range data.Webhooks { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "
NameStatusEndpointEventsPath PrefixesTokenActions
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var3 string + templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(hook.Name) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 51, Col: 63} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if hook.Enabled { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "Enabled") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "Disabled") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var4 string + templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(hook.Endpoint) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 59, Col: 65} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var5 string + templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(webhookFilterLabel(hook.EventTypes)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 60, Col: 81} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var6 string + templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(webhookFilterLabel(hook.PathPrefixes)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/notification_webhooks.templ`, Line: 61, Col: 83} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if hook.HasBearerToken { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "-") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "
Add Webhook
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + for _, et := range []string{"create", "update", "delete", "rename"} { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "
None selected sends every event type.
One per line. Empty sends events for every path.
Delivery settings
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = webhookNumberInput("webhookTimeout", "Timeout (s)", "10").Render(ctx, templ_7745c5c3_Buffer) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = webhookNumberInput("webhookRetries", "Max Retries", "3").Render(ctx, templ_7745c5c3_Buffer) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = webhookNumberInput("webhookBackoff", "Backoff (s)", "3").Render(ctx, templ_7745c5c3_Buffer) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = webhookNumberInput("webhookMaxBackoff", "Max Backoff (s)", "30").Render(ctx, templ_7745c5c3_Buffer) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = webhookNumberInput("webhookWorkers", "Workers", "5").Render(ctx, templ_7745c5c3_Buffer) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = webhookNumberInput("webhookBuffer", "Buffer Size", "10000").Render(ctx, templ_7745c5c3_Buffer) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + return nil + }) +} + +func webhookNumberInput(id string, label string, placeholder string) templ.Component { + return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { + templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context + if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { + return templ_7745c5c3_CtxErr + } + templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) + if !templ_7745c5c3_IsBuffer { + defer func() { + templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) + if templ_7745c5c3_Err == nil { + templ_7745c5c3_Err = templ_7745c5c3_BufErr + } + }() + } + ctx = templ.InitializeContext(ctx) + ctx = templ.ClearChildren(ctx) + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + return nil + }) +} + +func webhookFilterLabel(values []string) string { + if len(values) == 0 { + return "all" + } + return strings.Join(values, ", ") +} + +var _ = templruntime.GeneratedTemplate diff --git a/weed/admin/view/layout/layout.templ b/weed/admin/view/layout/layout.templ index ebf437d3a7..2041908631 100644 --- a/weed/admin/view/layout/layout.templ +++ b/weed/admin/view/layout/layout.templ @@ -288,6 +288,11 @@ templ Layout(view ViewContext, content templ.Component) { File Browser +
  • ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "\">File Browser
  • Webhooks
  • ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if isMQPage { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "Message Queue ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "Message Queue ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "Message Queue ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "Message Queue ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } if isMQPage { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "
    • ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "
      • ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if currentPath == "/mq/brokers" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "Brokers") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "Brokers") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "\">Brokers") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "
      • ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if currentPath == "/mq/topics" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "Topics") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "\">Brokers") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "
      • ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if currentPath == "/mq/topics" { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "Topics") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "\">Topics") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "Topics") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "
      ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "
  • WORKERS
    • ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if isDefaultWorkerPage { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "Brokers
    • Default") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "\">Topics
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, "
    WORKERS
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 83, "
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -961,108 +974,95 @@ func Layout(view ViewContext, content templ.Component) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 83, "
    © ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var54 string - templ_7745c5c3_Var54, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year())) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 419, Col: 60} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var54)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 84, " SeaweedFS Admin v") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 84, "
    © ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var55 string - templ_7745c5c3_Var55, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER) + templ_7745c5c3_Var55, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year())) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 419, Col: 102} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 424, Col: 60} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var55)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 85, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 85, " SeaweedFS Admin v") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var56 string + templ_7745c5c3_Var56, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 424, Col: 102} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var56)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 86, " ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if version.COMMIT != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 86, "(") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 87, "(") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var56 string - templ_7745c5c3_Var56, templ_7745c5c3_Err = templ.JoinStringErrs(version.COMMIT) + var templ_7745c5c3_Var57 string + templ_7745c5c3_Var57, templ_7745c5c3_Err = templ.JoinStringErrs(version.COMMIT) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 421, Col: 55} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 426, Col: 55} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var56)) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var57)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 87, ") ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 88, ") ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } if !strings.Contains(version.VERSION, "enterprise") { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 88, "• Enterprise Version Available") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 89, "• Enterprise Version Available") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 89, "
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 95, "\">") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -1120,33 +1133,33 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon ctx = templ.InitializeContext(ctx) ctx = templ.ClearChildren(ctx) prefix := dash.URLPrefixFromContext(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var64 string - templ_7745c5c3_Var64, templ_7745c5c3_Err = templ.JoinStringErrs(title) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 454, Col: 17} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var64)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 97, " - Login") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var65 string - templ_7745c5c3_Var65, templ_7745c5c3_Err = templruntime.NewAttributeExpression(prefix + "/static/favicon.ico").ResolveURL(ctx) + templ_7745c5c3_Var65, templ_7745c5c3_Err = templ.JoinStringErrs(title) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 455, Col: 58} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 459, Col: 17} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var65) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var65)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 98, "\" type=\"image/x-icon\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 98, " - Login") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -1154,27 +1167,14 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 99, "

    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 102, "\" rel=\"stylesheet\">

    Please sign in to continue

    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 103, "\" rel=\"stylesheet\">

    ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var70 string + templ_7745c5c3_Var70, templ_7745c5c3_Err = templ.JoinStringErrs(title) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 475, Col: 57} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var70)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 104, "

    Please sign in to continue

    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if errorMessage != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 104, "
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 105, "
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var70 string - templ_7745c5c3_Var70, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage) + var templ_7745c5c3_Var71 string + templ_7745c5c3_Var71, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 477, Col: 45} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 482, Col: 45} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var70)) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var71)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 105, "
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 106, "
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 106, "
    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 109, "\">
    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/command/scaffold/notification.toml b/weed/command/scaffold/notification.toml index 7d327984e4..fa85731787 100644 --- a/weed/command/scaffold/notification.toml +++ b/weed/command/scaffold/notification.toml @@ -70,6 +70,12 @@ sub_url = "rabbit://myqueue" # BEST FOR: Low to moderate traffic (< 100 events/second sustained) # FOR HIGH TRAFFIC: Consider using Kafka, SQS, or pull-based event logs instead # Documentation: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Notification-Webhook +# +# Webhook targets edited in the admin UI (stored in the filer at +# /etc/seaweedfs/notification_webhooks.json) always run and apply without a +# filer restart, whether or not this section is enabled. They are sent after +# the notification queue configured in this file, so a saturated target there +# also delays them. enabled = false endpoint = "https://your-server.com/webhook" # required: HTTP endpoint URL bearer_token = "" # optional: bearer token for authentication diff --git a/weed/filer/filer.go b/weed/filer/filer.go index c93b9045f7..0fa2c4399f 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -80,6 +80,7 @@ type Filer struct { RemoteStorage *FilerRemoteStorage BuildGuardedRemoteClient RemoteStorageClientBuilder AllowUntrustedRemoteEndpoints bool + webhooks managedWebhooks lazyFetchGroup singleflight.Group lazyListGroup singleflight.Group Dlm *lock_manager.DistributedLockManager diff --git a/weed/filer/filer_notification_webhooks.go b/weed/filer/filer_notification_webhooks.go new file mode 100644 index 0000000000..8b570b7c00 --- /dev/null +++ b/weed/filer/filer_notification_webhooks.go @@ -0,0 +1,146 @@ +package filer + +import ( + "bytes" + "context" + "errors" + "sync" + "sync/atomic" + "time" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/notification/webhook" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +const ( + NotificationWebhooksConfName = "notification_webhooks.json" + notificationWebhooksDrain = 30 * time.Second +) + +var notificationWebhooksRetry = 5 * time.Second + +var NotificationWebhooksConfPath = string(util.NewFullPath(DirectoryEtcSeaweedFS, NotificationWebhooksConfName)) + +// managedWebhooks runs the webhook targets edited from the admin UI, next to +// whatever notification.toml configures. +type managedWebhooks struct { + reloadMu sync.Mutex + applied []byte + retry *time.Timer + queue atomic.Pointer[webhook.Queue] +} + +func (f *Filer) LoadNotificationWebhooks() { + f.reloadNotificationWebhooks() + go func() { + <-f.deletionQuit + f.webhooks.reloadMu.Lock() + defer f.webhooks.reloadMu.Unlock() + if f.webhooks.retry != nil { + f.webhooks.retry.Stop() + } + if q := f.webhooks.queue.Swap(nil); q != nil { + go q.Close(notificationWebhooksDrain) + } + }() +} + +func (f *Filer) maybeReloadNotificationWebhooks(event *filer_pb.SubscribeMetadataResponse) { + if event == nil || event.EventNotification == nil { + return + } + n := event.EventNotification + touched := n.OldEntry != nil && n.OldEntry.Name == NotificationWebhooksConfName && event.Directory == DirectoryEtcSeaweedFS + if n.NewEntry != nil && n.NewEntry.Name == NotificationWebhooksConfName && filer_pb.MetadataEventTargetDirectory(event) == DirectoryEtcSeaweedFS { + touched = true + } + if touched { + f.reloadNotificationWebhooks() + } +} + +// reloadNotificationWebhooks reads the stored targets rather than the event's +// copy, so concurrent change events cannot apply an older version last. +func (f *Filer) reloadNotificationWebhooks() { + f.webhooks.reloadMu.Lock() + defer f.webhooks.reloadMu.Unlock() + select { + case <-f.deletionQuit: + return + default: + } + + var content []byte + entry, err := f.FindEntry(context.Background(), util.FullPath(NotificationWebhooksConfPath)) + switch { + case err == filer_pb.ErrNotFound: + case err != nil: + glog.Errorf("read %s: %v", NotificationWebhooksConfPath, err) + f.retryNotificationWebhooks() + return + case len(entry.Content) > 0: + content = entry.Content + default: + if content, err = f.readEntry(entry.GetChunks(), entry.Size()); err != nil { + glog.Errorf("read %s: %v", NotificationWebhooksConfPath, err) + f.retryNotificationWebhooks() + return + } + } + + if f.webhooks.applied != nil && bytes.Equal(content, f.webhooks.applied) { + return + } + conf, err := webhook.ParseManagedConfig(content) + if err != nil { + glog.Errorf("keep current webhook targets, %s: %v", NotificationWebhooksConfPath, err) + return + } + queue, err := webhook.NewManagedQueue(conf) + if err != nil { + glog.Errorf("keep current webhook targets, start %s: %v", NotificationWebhooksConfPath, err) + return + } + glog.V(0).Infof("apply %d webhook targets from %s", len(conf.Targets), NotificationWebhooksConfPath) + f.webhooks.applied = append([]byte{}, content...) + if old := f.webhooks.queue.Swap(queue); old != nil { + go old.Close(notificationWebhooksDrain) + } +} + +// retryNotificationWebhooks keeps at most one pending retry, since no later +// event replays the change whose read failed. +func (f *Filer) retryNotificationWebhooks() { + if f.webhooks.retry != nil { + f.webhooks.retry.Stop() + } + f.webhooks.retry = time.AfterFunc(notificationWebhooksRetry, f.reloadNotificationWebhooks) +} + +// sendToNotificationWebhooks hands a send that raced a reload to the queue +// that replaced the closing one. +func (f *Filer) sendToNotificationWebhooks(key string, message *filer_pb.EventNotification) { + for { + q := f.webhooks.queue.Load() + if q == nil { + return + } + err := q.SendMessage(key, message) + if errors.Is(err, webhook.ErrQueueClosed) && f.webhooks.queue.Load() != q { + continue + } + if err != nil { + glog.Error(err) + } + return + } +} + +// isNotificationWebhooksConf keeps the stored targets, bearer tokens +// included, out of the events sent to notification queues. +func isNotificationWebhooksConf(oldEntry, newEntry *Entry) bool { + return (oldEntry != nil && string(oldEntry.FullPath) == NotificationWebhooksConfPath) || + (newEntry != nil && string(newEntry.FullPath) == NotificationWebhooksConfPath) +} diff --git a/weed/filer/filer_notification_webhooks_test.go b/weed/filer/filer_notification_webhooks_test.go new file mode 100644 index 0000000000..9e9f47b62a --- /dev/null +++ b/weed/filer/filer_notification_webhooks_test.go @@ -0,0 +1,176 @@ +package filer + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +func TestManagedNotificationWebhooks(t *testing.T) { + var mu sync.Mutex + var keys []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct{ Key string } + _ = json.NewDecoder(r.Body).Decode(&body) + mu.Lock() + keys = append(keys, body.Key) + mu.Unlock() + })) + defer server.Close() + received := func() []string { + mu.Lock() + defer mu.Unlock() + return append([]string(nil), keys...) + } + + store := newStubFilerStore() + f := newTestFiler(t, store, nil) + t.Cleanup(func() { + if q := f.webhooks.queue.Load(); q != nil { + q.Close(time.Second) + } + }) + + saveConf := func(content string) { + store.entries[NotificationWebhooksConfPath] = &Entry{FullPath: util.FullPath(NotificationWebhooksConfPath), Content: []byte(content)} + } + confEvent := &filer_pb.SubscribeMetadataResponse{ + Directory: DirectoryEtcSeaweedFS, + EventNotification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: NotificationWebhooksConfName}}, + } + + f.LoadNotificationWebhooks() + if f.webhooks.queue.Load() != nil { + t.Fatal("expected no webhook queue without stored targets") + } + + saveConf(fmt.Sprintf(`{"targets":[{"name":"data","enabled":true,"endpoint":%q,"bearer_token":"tok","path_prefixes":["/"]}]}`, server.URL)) + f.maybeReloadNotificationWebhooks(confEvent) + if f.webhooks.queue.Load() == nil { + t.Fatal("expected a webhook queue after the targets were stored") + } + + ctx := context.Background() + f.NotifyUpdateEvent(ctx, nil, store.entries[NotificationWebhooksConfPath], false, false, nil) + f.NotifyUpdateEvent(ctx, nil, &Entry{FullPath: "/data/a.txt"}, false, false, nil) + waitForWebhook(t, func() bool { return len(received()) == 1 }) + time.Sleep(200 * time.Millisecond) + if got := received(); len(got) != 1 || got[0] != "/data/a.txt" { + t.Fatalf("expected only /data/a.txt to be delivered, got %v", got) + } + + running := f.webhooks.queue.Load() + f.maybeReloadNotificationWebhooks(confEvent) + if f.webhooks.queue.Load() != running { + t.Fatal("unchanged targets must keep the running queue") + } + saveConf(`{"targets":[{"name":"bad name","enabled":true,"endpoint":"https://example.com"}]}`) + f.maybeReloadNotificationWebhooks(confEvent) + if f.webhooks.queue.Load() != running { + t.Fatal("invalid targets must keep the running queue") + } + + delete(store.entries, NotificationWebhooksConfPath) + f.maybeReloadNotificationWebhooks(&filer_pb.SubscribeMetadataResponse{ + Directory: DirectoryEtcSeaweedFS, + EventNotification: &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: NotificationWebhooksConfName}}, + }) + if f.webhooks.queue.Load() != nil { + t.Fatal("expected deleting the stored targets to stop the queue") + } +} + +func waitForWebhook(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatal("timed out") + } + time.Sleep(20 * time.Millisecond) + } +} + +func TestManagedNotificationWebhooksRetryAfterReadError(t *testing.T) { + retry := notificationWebhooksRetry + t.Cleanup(func() { notificationWebhooksRetry = retry }) + notificationWebhooksRetry = 10 * time.Millisecond + + store := newStubFilerStore() + f := newTestFiler(t, store, nil) + t.Cleanup(func() { + f.webhooks.reloadMu.Lock() + if f.webhooks.retry != nil { + f.webhooks.retry.Stop() + } + f.webhooks.reloadMu.Unlock() + if q := f.webhooks.queue.Load(); q != nil { + q.Close(time.Second) + } + }) + + store.mu.Lock() + store.entries[NotificationWebhooksConfPath] = &Entry{ + FullPath: util.FullPath(NotificationWebhooksConfPath), + Content: []byte(`{"targets":[{"name":"data","enabled":true,"endpoint":"http://127.0.0.1:1/hook"}]}`), + } + store.findErr = errors.New("store unavailable") + store.mu.Unlock() + + f.LoadNotificationWebhooks() + if f.webhooks.queue.Load() != nil { + t.Fatal("expected no webhook queue while the stored targets cannot be read") + } + + store.mu.Lock() + store.findErr = nil + store.mu.Unlock() + deadline := time.Now().Add(5 * time.Second) + for f.webhooks.queue.Load() == nil { + if time.Now().After(deadline) { + t.Fatal("stored webhook targets were not applied after the read recovered") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestManagedNotificationWebhooksStopOnShutdown(t *testing.T) { + store := newStubFilerStore() + f := newTestFiler(t, store, nil) + store.entries[NotificationWebhooksConfPath] = &Entry{ + FullPath: util.FullPath(NotificationWebhooksConfPath), + Content: []byte(`{"targets":[{"name":"data","enabled":true,"endpoint":"http://127.0.0.1:1/hook"}]}`), + } + + f.LoadNotificationWebhooks() + if f.webhooks.queue.Load() == nil { + t.Fatal("expected a webhook queue for the stored targets") + } + + close(f.deletionQuit) + deadline := time.Now().Add(5 * time.Second) + for f.webhooks.queue.Load() != nil { + if time.Now().After(deadline) { + t.Fatal("webhook queue still installed after shutdown") + } + time.Sleep(10 * time.Millisecond) + } + + store.entries[NotificationWebhooksConfPath] = &Entry{ + FullPath: util.FullPath(NotificationWebhooksConfPath), + Content: []byte(`{"targets":[{"name":"late","enabled":true,"endpoint":"http://127.0.0.1:1/late"}]}`), + } + f.reloadNotificationWebhooks() + if f.webhooks.queue.Load() != nil { + t.Fatal("a reload after shutdown installed a webhook queue") + } +} diff --git a/weed/filer/filer_notify.go b/weed/filer/filer_notify.go index abb49e046b..baf646537d 100644 --- a/weed/filer/filer_notify.go +++ b/weed/filer/filer_notify.go @@ -68,12 +68,15 @@ func (f *Filer) notifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry defer f.metaLogInflight.done(event.TsNs) eventNotification := event.EventNotification - if notification.Queue != nil { - glog.V(3).Infof("notifying entry update %v", fullpath) - if err := notification.Queue.SendMessage(fullpath, eventNotification); err != nil { - // throw message - glog.Error(err) + if !isNotificationWebhooksConf(oldEntry, newEntry) { + if notification.Queue != nil { + glog.V(3).Infof("notifying entry update %v", fullpath) + if err := notification.Queue.SendMessage(fullpath, eventNotification); err != nil { + // throw message + glog.Error(err) + } } + f.sendToNotificationWebhooks(fullpath, eventNotification) } f.logMetaEvent(ctx, event) diff --git a/weed/filer/filer_on_meta_event.go b/weed/filer/filer_on_meta_event.go index 1ecfdcfc03..8735bc21af 100644 --- a/weed/filer/filer_on_meta_event.go +++ b/weed/filer/filer_on_meta_event.go @@ -12,6 +12,7 @@ import ( // onMetadataChangeEvent is triggered after filer processed change events from local or remote filers func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse) { f.maybeReloadFilerConfiguration(event) + f.maybeReloadNotificationWebhooks(event) f.maybeReloadRemoteStorageConfigurationAndMapping(event) f.onBucketEvents(event) f.onEmptyFolderCleanupEvents(event) diff --git a/weed/notification/webhook/managed.go b/weed/notification/webhook/managed.go new file mode 100644 index 0000000000..a382d6e52e --- /dev/null +++ b/weed/notification/webhook/managed.go @@ -0,0 +1,150 @@ +package webhook + +import ( + "encoding/json" + "fmt" + "net/url" + "regexp" + "strings" +) + +// ManagedTarget is a webhook target kept in the filer and edited from the +// admin UI. Zero numeric settings take the same defaults as notification.toml. +type ManagedTarget struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + Endpoint string `json:"endpoint"` + BearerToken string `json:"bearer_token,omitempty"` + EventTypes []string `json:"event_types,omitempty"` + PathPrefixes []string `json:"path_prefixes,omitempty"` + TimeoutSeconds int `json:"timeout_seconds,omitempty"` + MaxRetries int `json:"max_retries,omitempty"` + BackoffSeconds int `json:"backoff_seconds,omitempty"` + MaxBackoffSeconds int `json:"max_backoff_seconds,omitempty"` + Workers int `json:"workers,omitempty"` + BufferSize int `json:"buffer_size,omitempty"` +} + +type ManagedConfig struct { + Targets []ManagedTarget `json:"targets"` +} + +var managedTargetName = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) + +func ParseManagedConfig(data []byte) (*ManagedConfig, error) { + c := &ManagedConfig{} + if len(strings.TrimSpace(string(data))) == 0 { + return c, nil + } + if err := json.Unmarshal(data, c); err != nil { + return nil, fmt.Errorf("parse webhook targets: %w", err) + } + if err := c.Validate(); err != nil { + return nil, err + } + return c, nil +} + +func (c *ManagedConfig) Validate() error { + seen := make(map[string]bool, len(c.Targets)) + for i := range c.Targets { + t := &c.Targets[i] + if seen[t.Name] { + return fmt.Errorf("duplicate webhook target %q", t.Name) + } + seen[t.Name] = true + if err := t.Validate(); err != nil { + return err + } + } + return nil +} + +func (t *ManagedTarget) Validate() error { + if !managedTargetName.MatchString(t.Name) { + return fmt.Errorf("webhook target name %q must be 1-64 letters, digits, '-' or '_'", t.Name) + } + u, err := url.Parse(t.Endpoint) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return fmt.Errorf("webhook target %q: endpoint must be an http or https URL", t.Name) + } + for _, et := range t.EventTypes { + if !eventType(et).valid() { + return fmt.Errorf("webhook target %q: invalid event type %q", t.Name, et) + } + } + for _, p := range t.PathPrefixes { + if !strings.HasPrefix(p, "/") { + return fmt.Errorf("webhook target %q: path prefix %q must start with '/'", t.Name, p) + } + } + for field, v := range t.settings() { + if v < 0 { + return fmt.Errorf("webhook target %q: %s must not be negative", t.Name, field) + } + } + if err := t.config().validate(); err != nil { + return fmt.Errorf("webhook target %q: %w", t.Name, err) + } + return nil +} + +func (t *ManagedTarget) settings() map[string]int { + return map[string]int{ + "timeout_seconds": t.TimeoutSeconds, + "max_retries": t.MaxRetries, + "backoff_seconds": t.BackoffSeconds, + "max_backoff_seconds": t.MaxBackoffSeconds, + "workers": t.Workers, + "buffer_size": t.BufferSize, + } +} + +func (t *ManagedTarget) config() *config { + c := defaultConfig() + c.endpoint = t.Endpoint + c.authBearerToken = t.BearerToken + c.eventTypes = t.EventTypes + c.pathPrefixes = t.PathPrefixes + for dst, v := range map[*int]int{ + &c.timeoutSeconds: t.TimeoutSeconds, + &c.maxRetries: t.MaxRetries, + &c.backoffSeconds: t.BackoffSeconds, + &c.maxBackoffSeconds: t.MaxBackoffSeconds, + &c.nWorkers: t.Workers, + &c.bufferSize: t.BufferSize, + } { + if v > 0 { + *dst = v + } + } + return c +} + +// NewManagedQueue starts delivering to the enabled targets; it returns nil +// when none is enabled. +func NewManagedQueue(c *ManagedConfig) (*Queue, error) { + if err := c.Validate(); err != nil { + return nil, err + } + var targets []*target + for i := range c.Targets { + if !c.Targets[i].Enabled { + continue + } + t, err := newTarget(c.Targets[i].Name, c.Targets[i].config()) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + if len(targets) == 0 { + return nil, nil + } + w := &Queue{managed: true} + if err := w.initialize(targets); err != nil { + w.cancel() + return nil, err + } + return w, nil +} diff --git a/weed/notification/webhook/managed_test.go b/weed/notification/webhook/managed_test.go new file mode 100644 index 0000000000..d5b55ef05f --- /dev/null +++ b/weed/notification/webhook/managed_test.go @@ -0,0 +1,165 @@ +package webhook + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" +) + +func TestParseManagedConfig(t *testing.T) { + c, err := ParseManagedConfig(nil) + if err != nil || len(c.Targets) != 0 { + t.Fatalf("empty content: got %+v, %v", c, err) + } + + c, err = ParseManagedConfig([]byte(`{"targets":[{"name":"ingest","enabled":true,"endpoint":"https://example.com/hook","event_types":["create"],"path_prefixes":["/buckets/a"],"max_retries":5}]}`)) + if err != nil { + t.Fatalf("ParseManagedConfig: %v", err) + } + cfg := c.Targets[0].config() + if cfg.maxRetries != 5 || cfg.timeoutSeconds != 10 || cfg.nWorkers != 5 || cfg.bufferSize != 10_000 { + t.Fatalf("unexpected settings %+v", cfg) + } + + for name, body := range map[string]string{ + "bad name": `{"targets":[{"name":"a b","endpoint":"https://example.com"}]}`, + "duplicate": `{"targets":[{"name":"a","endpoint":"https://example.com"},{"name":"a","endpoint":"https://example.com"}]}`, + "no scheme": `{"targets":[{"name":"a","endpoint":"example.com/hook"}]}`, + "ftp endpoint": `{"targets":[{"name":"a","endpoint":"ftp://example.com/hook"}]}`, + "event type": `{"targets":[{"name":"a","endpoint":"https://example.com","event_types":["chmod"]}]}`, + "relative path": `{"targets":[{"name":"a","endpoint":"https://example.com","path_prefixes":["buckets"]}]}`, + "timeout range": `{"targets":[{"name":"a","endpoint":"https://example.com","timeout_seconds":301}]}`, + "neg timeout": `{"targets":[{"name":"a","endpoint":"https://example.com","timeout_seconds":-1}]}`, + "neg workers": `{"targets":[{"name":"a","endpoint":"https://example.com","workers":-1}]}`, + "not json": `targets`, + } { + if _, err := ParseManagedConfig([]byte(body)); err == nil { + t.Errorf("%s: expected an error", name) + } + } +} + +func TestManagedQueueDeliversAndDrainsOnClose(t *testing.T) { + var hits atomic.Int32 + var auth atomic.Value + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + auth.Store(r.Header.Get("Authorization")) + <-release + hits.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + unblock := sync.OnceFunc(func() { close(release) }) + defer unblock() + + q, err := NewManagedQueue(&ManagedConfig{Targets: []ManagedTarget{ + {Name: "on", Enabled: true, Endpoint: server.URL, BearerToken: "secret", PathPrefixes: []string{"/data/"}}, + {Name: "off", Endpoint: "https://example.com/never"}, + }}) + if err != nil { + t.Fatalf("NewManagedQueue: %v", err) + } + if len(q.targets) != 1 { + t.Fatalf("expected only the enabled target, got %d", len(q.targets)) + } + + msg := &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "f.txt"}} + for _, key := range []string{"/data/f.txt", "/other/f.txt"} { + if err := q.SendMessage(key, msg); err != nil { + t.Fatalf("SendMessage(%q): %v", key, err) + } + } + + closed := make(chan struct{}) + go func() { + q.Close(10 * time.Second) + close(closed) + }() + select { + case <-closed: + t.Fatal("Close returned before the in-flight delivery finished") + case <-time.After(300 * time.Millisecond): + } + unblock() + select { + case <-closed: + case <-time.After(10 * time.Second): + t.Fatal("Close did not return after the delivery finished") + } + + if got := hits.Load(); got != 1 { + t.Fatalf("expected 1 delivery, got %d", got) + } + if got, _ := auth.Load().(string); !strings.HasSuffix(got, "secret") { + t.Fatalf("expected the bearer token, got %q", got) + } +} + +func TestManagedQueueNoEnabledTargets(t *testing.T) { + q, err := NewManagedQueue(&ManagedConfig{Targets: []ManagedTarget{{Name: "off", Endpoint: "https://example.com"}}}) + if err != nil || q != nil { + t.Fatalf("expected no queue, got %v, %v", q, err) + } +} + +func TestManagedQueueCloseWaitsForBlockedSender(t *testing.T) { + var hits atomic.Int32 + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + hits.Add(1) + })) + defer server.Close() + unblock := sync.OnceFunc(func() { close(release) }) + defer unblock() + + q, err := NewManagedQueue(&ManagedConfig{Targets: []ManagedTarget{ + {Name: "slow", Enabled: true, Endpoint: server.URL, Workers: 1, BufferSize: 100}, + }}) + if err != nil { + t.Fatalf("NewManagedQueue: %v", err) + } + msg := &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "f.txt"}} + for i := 0; i < 100; i++ { + if err := q.SendMessage("/a", msg); err != nil { + t.Fatalf("SendMessage: %v", err) + } + } + blocked := make(chan error, 1) + go func() { blocked <- q.SendMessage("/b", msg) }() + for q.senders.Load() == 0 { + time.Sleep(time.Millisecond) + } + + closed := make(chan struct{}) + go func() { + q.Close(10 * time.Second) + close(closed) + }() + for !q.closing.Load() { + time.Sleep(time.Millisecond) + } + if err := q.SendMessage("/c", msg); err != ErrQueueClosed { + t.Fatalf("expected ErrQueueClosed after Close started, got %v", err) + } + + unblock() + if err := <-blocked; err != nil { + t.Fatalf("blocked sender: %v", err) + } + select { + case <-closed: + case <-time.After(10 * time.Second): + t.Fatal("Close did not return") + } + if got := hits.Load(); got != 101 { + t.Fatalf("expected both accepted events delivered, got %d", got) + } +} diff --git a/weed/notification/webhook/types.go b/weed/notification/webhook/types.go index 1567e832f2..6309bb7009 100644 --- a/weed/notification/webhook/types.go +++ b/weed/notification/webhook/types.go @@ -69,10 +69,8 @@ type config struct { pathPrefixes []string } -func newConfigWithDefaults(configuration util.Configuration, prefix string) *config { - c := &config{ - endpoint: configuration.GetString(prefix + "endpoint"), - authBearerToken: configuration.GetString(prefix + "bearer_token"), +func defaultConfig() *config { + return &config{ timeoutSeconds: 10, maxRetries: 3, backoffSeconds: 3, @@ -80,6 +78,12 @@ func newConfigWithDefaults(configuration util.Configuration, prefix string) *con nWorkers: 5, bufferSize: 10_000, } +} + +func newConfigWithDefaults(configuration util.Configuration, prefix string) *config { + c := defaultConfig() + c.endpoint = configuration.GetString(prefix + "endpoint") + c.authBearerToken = configuration.GetString(prefix + "bearer_token") if bufferSize := configuration.GetInt(prefix + "buffer_size"); bufferSize > 0 { c.bufferSize = bufferSize diff --git a/weed/notification/webhook/webhook_queue.go b/weed/notification/webhook/webhook_queue.go index afaa326245..3172dcb06f 100644 --- a/weed/notification/webhook/webhook_queue.go +++ b/weed/notification/webhook/webhook_queue.go @@ -7,6 +7,7 @@ import ( "sort" "strings" "sync" + "sync/atomic" "time" "github.com/ThreeDotsLabs/watermill" @@ -40,9 +41,9 @@ type target struct { func newTarget(name string, cfg *config) (*target, error) { t := &target{ - name: name, - config: cfg, - filter: newFilter(cfg), + name: name, + config: cfg, + filter: newFilter(cfg), sem: make(chan struct{}, cfg.nWorkers), inbox: make(chan *message.Message, cfg.bufferSize), pending: make(chan struct{}, cfg.bufferSize), @@ -74,10 +75,18 @@ type Queue struct { queueChannel *gochannel.GoChannel targets []*target - ctx context.Context - cancel context.CancelFunc + ctx context.Context + cancel context.CancelFunc + stopped chan struct{} + // managed queues are replaced on reload and stopped with Close + managed bool + closing atomic.Bool + senders atomic.Int64 } +// ErrQueueClosed is returned by SendMessage once Close has started. +var ErrQueueClosed = errors.New("webhook queue closed") + func (w *Queue) GetName() string { return queueName } @@ -138,6 +147,11 @@ func buildTargets(configuration util.Configuration, prefix string) ([]*target, e } func (w *Queue) SendMessage(key string, msg proto.Message) error { + w.senders.Add(1) + defer w.senders.Add(-1) + if w.closing.Load() { + return ErrQueueClosed + } eventNotification, ok := msg.(*filer_pb.EventNotification) if !ok { return nil @@ -225,6 +239,7 @@ func (w *Queue) Initialize(configuration util.Configuration, prefix string) erro func (w *Queue) initialize(targets []*target) error { w.ctx, w.cancel = context.WithCancel(context.Background()) + w.stopped = make(chan struct{}) w.targets = targets if err := w.setupWatermillQueue(targets); err != nil { @@ -281,7 +296,9 @@ func (w *Queue) setupWatermillQueue(targets []*target) error { return fmt.Errorf("failed to create poison queue: %w", err) } - router.AddPlugin(plugin.SignalsHandler) + if !w.managed { + router.AddPlugin(plugin.SignalsHandler) + } // outermost: the reservation SendMessage took releases once the whole // chain (retries, dead letter) is done, bounding in-flight deliveries. // poison wraps retry so the delivery is retried before dead-lettering. @@ -329,11 +346,37 @@ func (w *Queue) setupWatermillQueue(targets []*target) error { go func() { wg.Wait() w.cancel() + close(w.stopped) }() return nil } +// Close stops the queue once every sender already inside SendMessage has +// returned and every target has delivered or dead-lettered what it holds, or +// when drainTimeout passes. +func (w *Queue) Close(drainTimeout time.Duration) { + w.closing.Store(true) + deadline := time.Now().Add(drainTimeout) + for (w.senders.Load() > 0 || !w.drained()) && time.Now().Before(deadline) { + time.Sleep(50 * time.Millisecond) + } + w.cancel() + <-w.stopped + if err := w.queueChannel.Close(); err != nil { + glog.Warningf("close webhook queue: %v", err) + } +} + +func (w *Queue) drained() bool { + for _, t := range w.targets { + if len(t.pending) > 0 { + return false + } + } + return true +} + // forwardTargetMessages publishes each queued event to the target's topic in // order. Publish detaches its delivery, so the pending reservation is what // keeps a saturated target from queuing without bound. diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index c520509195..5bbbc733a1 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -420,6 +420,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) } fs.filer.LoadFilerConf() + fs.filer.LoadNotificationWebhooks() fs.filer.LoadRemoteStorageConfAndMapping() diff --git a/weed/shell/command_fs_meta_notify.go b/weed/shell/command_fs_meta_notify.go index 911bff7dc4..af5a7dcfe3 100644 --- a/weed/shell/command_fs_meta_notify.go +++ b/weed/shell/command_fs_meta_notify.go @@ -5,6 +5,7 @@ import ( "fmt" "io" + "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/notification" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/util" @@ -54,6 +55,9 @@ func (c *commandFsMetaNotify) Do(args []string, commandEnv *CommandEnv, writer i err = filer_pb.TraverseBfs(context.Background(), commandEnv, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) error { + if string(parentPath.Child(entry.Name)) == filer.NotificationWebhooksConfPath { + return nil + } if entry.IsDirectory { dirCount++ } else {