mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-25 14:30:43 +01:00
Chris LuandDevin
be081b49b7
* notification: honor the config prefix in the multi-queue guard validateOneEnabledQueue checked "<name>.enabled" without the notification prefix, so the fatal-on-two-queues guard never saw the real keys and could never fire. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * notification: support named webhook targets under [notification.webhook.<name>] The webhook queue only accepted one endpoint, so events could not be routed to different services by path prefix. Each enabled [notification.webhook.<name>] subsection now spins up its own target — own filter, http client, retry middleware and worker budget — published on a per-target topic so every matching endpoint gets each event exactly once. The flat [notification.webhook] section keeps working as the unnamed target, and named subsections alone now activate the queue. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * notification: require enabled for the flat webhook target buildTargets included the flat section whenever an endpoint was set, so a disabled [notification.webhook] block — which the scaffold ships with a placeholder endpoint — would start receiving events as soon as any named target activated the queue. * notification: resolve named target enabled flags through config namedTargetPrefixes read the raw map value, which never sees environment variable overrides; GetBool resolves WEED_* env bindings, so a named target disabled in the file can now be switched on by WEED_NOTIFICATION_WEBHOOK_<NAME>_ENABLED. * notification: cancel the webhook queue only after all workers stop Each target's router goroutine deferred the shared cancel, so the first worker to exit tore down the queue context for every other target and the dead letter logger. * notification: isolate webhook targets behind per-target queues SendMessage published to each target's topic inline, so a subscriber channel that filled up blocked every later target and the filer's notification path with it. Each target now drains its own inbox through a forwarder goroutine; SendMessage fails fast on a full inbox, matching the "throw message" contract the filer already applies to errors. * notification: name the webhook target in failure and dead letter logs Every target dead-letters to the same topic, so without the name there is no way to tell which endpoint failed or needs a replay. * notification: assert delivery in the named targets test The test only exercised SendMessage; it now records requests on both endpoints and checks matching, non-matching, and unrouted keys reach the right servers exactly once. * notification: wait for the webhook router before accepting events * notification: apply backpressure instead of dropping a full webhook inbox * notification: bound queued and in-flight webhook deliveries per target * notification: release the pending token when shutdown races the enqueue A SendMessage that wins the pending select as the queue is canceling could block forever on a full, undrained inbox. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * notification: stop waiting for a router that failed to start A router that exits before reaching Running() would leave initialize blocked on a context nothing had canceled yet. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * notification: route webhook events past saturated targets A full pending reservation made SendMessage wait on the slow target before it ever reached the healthy ones after it, coupling every endpoint's latency to the most congested one. Targets that can take a reservation now get the event immediately; saturated targets defer to a second pass that still blocks for a slot, so backpressure is kept and no event is dropped. * notification: retry webhook deliveries before dead-lettering PoisonQueue must wrap the retry middleware: added after it, the poison middleware intercepts the first handler error, republishes to the dead letter topic, and returns nil — the retry chain never sees a failure, so max_retries/backoff settings were dead code and every delivery failed straight to dead letter on the first attempt. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
97 lines
4.4 KiB
TOML
97 lines
4.4 KiB
TOML
# A sample TOML config file for SeaweedFS filer store
|
||
# Used by both "weed filer" or "weed server -filer" and "weed filer.replicate"
|
||
# Put this file to one of the location, with descending priority
|
||
# ./notification.toml
|
||
# $HOME/.seaweedfs/notification.toml
|
||
# /etc/seaweedfs/notification.toml
|
||
|
||
####################################################
|
||
# notification
|
||
# send and receive filer updates for each file to an external message queue
|
||
####################################################
|
||
[notification.log]
|
||
# this is only for debugging purpose and does not work with "weed filer.replicate"
|
||
enabled = false
|
||
|
||
|
||
[notification.kafka]
|
||
enabled = false
|
||
hosts = [
|
||
"localhost:9092"
|
||
]
|
||
topic = "seaweedfs_filer"
|
||
# event_types = ["create", "update", "delete", "rename"] # optional: filter by event types (default: all)
|
||
offsetFile = "./last.offset"
|
||
offsetSaveIntervalSeconds = 10
|
||
# SASL Authentication
|
||
sasl_enabled = false
|
||
sasl_mechanism = "PLAIN" # PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
|
||
sasl_username = ""
|
||
sasl_password = ""
|
||
# TLS/SSL
|
||
tls_enabled = false
|
||
tls_ca_cert = "" # path to CA certificate PEM file
|
||
tls_client_cert = "" # path to client certificate PEM file (for mTLS)
|
||
tls_client_key = "" # path to client private key PEM file (for mTLS)
|
||
tls_insecure_skip_verify = false
|
||
|
||
|
||
[notification.aws_sqs]
|
||
# experimental, let me know if it works
|
||
enabled = false
|
||
aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
|
||
aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials).
|
||
region = "us-east-2"
|
||
sqs_queue_name = "my_filer_queue" # an existing queue name
|
||
|
||
|
||
[notification.google_pub_sub]
|
||
# read credentials doc at https://cloud.google.com/docs/authentication/getting-started
|
||
enabled = false
|
||
google_application_credentials = "/path/to/x.json" # path to json credential file
|
||
project_id = "" # an existing project id
|
||
topic = "seaweedfs_filer_topic" # a topic, auto created if does not exists
|
||
|
||
[notification.gocdk_pub_sub]
|
||
# The Go Cloud Development Kit (https://gocloud.dev).
|
||
# PubSub API (https://godoc.org/gocloud.dev/pubsub).
|
||
# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ.
|
||
enabled = false
|
||
# This URL will Dial the RabbitMQ server at the URL in the environment
|
||
# variable RABBIT_SERVER_URL and open the exchange "myexchange".
|
||
# The exchange must have already been created by some other means, like
|
||
# the RabbitMQ management plugin. Сreate myexchange of type fanout and myqueue then
|
||
# create binding myexchange => myqueue
|
||
topic_url = "rabbit://myexchange"
|
||
sub_url = "rabbit://myqueue"
|
||
|
||
[notification.webhook]
|
||
# Send file system events to HTTP webhook endpoints (push model)
|
||
# BEST FOR: Low to moderate traffic (< 100 events/second sustained)
|
||
# FOR HIGH TRAFFIC: Consider using Kafka, SQS, or pull-based event logs instead
|
||
# Documentation: https://github.com/seaweedfs/seaweedfs/wiki/Filer-Notification-Webhook
|
||
enabled = false
|
||
endpoint = "https://your-server.com/webhook" # required: HTTP endpoint URL
|
||
bearer_token = "" # optional: bearer token for authentication
|
||
timeout_seconds = 10 # optional: HTTP timeout (default: 10, range: 1-300)
|
||
max_retries = 3 # optional: retry attempts (default: 3, range: 0-10)
|
||
backoff_seconds = 3 # optional: initial backoff delay (default: 3, range: 1-60)
|
||
max_backoff_seconds = 30 # optional: max backoff delay (default: 30, range: backoff_seconds-300)
|
||
workers = 5 # optional: concurrent workers (default: 5, range: 1-100)
|
||
buffer_size = 10000 # optional: event buffer size (default: 10000, range: 100-1000000)
|
||
# event_types = ["create", "update", "delete", "rename"] # optional: filter by event types (default: all)
|
||
# path_prefixes = ["/important", "/data"] # optional: filter by path prefixes (default: all)
|
||
#
|
||
# Additional endpoints can be declared as named subsections, each with its own
|
||
# endpoint, filters, retry and worker settings, e.g.:
|
||
# [notification.webhook.file_ingress]
|
||
# enabled = true
|
||
# endpoint = "https://ingress.example.com/hook"
|
||
# event_types = ["create"]
|
||
# path_prefixes = ["/buckets/uploads/data"]
|
||
#
|
||
# [notification.webhook.metadata_extractor]
|
||
# enabled = true
|
||
# endpoint = "https://metadata.example.com/hook"
|
||
# path_prefixes = ["/buckets/repository"]
|