From 0d3f5a220c8870835d60ca2824135c85c3a7959b Mon Sep 17 00:00:00 2001 From: Peter Dodd Date: Wed, 18 Nov 2026 01:06:06 +0000 Subject: [PATCH] s3: keep serving for -preStopSeconds after SIGTERM (#12703) * s3: keep serving for -preStopSeconds after SIGTERM On SIGTERM the gateway closed its listeners at once, so requests a load balancer or Kubernetes Service still routed to it failed until the endpoint was removed. With -preStopSeconds set, the gateway makes /status, /healthz and /readyz return 503, keeps serving for that many seconds, then runs the existing graceful shutdown. 0 (the default) keeps the old behavior. The flag is registered for weed s3 and as -s3.preStopSeconds for weed filer, weed server and weed mini. A cancelled weed mini cluster context ends the wait early. The Helm chart passes s3.preStopSeconds and filer.s3.preStopSeconds through. Co-Authored-By: Claude Opus 5.5 * s3: assert the preStopSeconds chart render under set -e, drop timing from the probe test Co-Authored-By: Claude Opus 5.5 * s3: wait before weed mini cancels its clients; size grace periods in the chart weed mini's interrupt hook runs before the S3 one and cancels the S3 gateway's shutdownCtx, so the pre-stop wait never happened there. The wait is now also registered with beforeMiniClientsShutdown, which runs before that cancel, and a sync.OnceFunc keeps the interrupt hook from waiting a second time. Outside weed mini the registration is a no-op. Helm: the s3 deployment fails to render when terminationGracePeriodSeconds is below preStopSeconds + 15. The filer statefulset and the all-in-one deployment, whose grace period is fixed at 60s, add preStopSeconds to it. allInOne.s3.preStopSeconds is passed through, inheriting s3.preStopSeconds. Co-Authored-By: Claude Opus 5.5 * s3: keep 64-bit counters aligned on 386; let allInOne.s3.preStopSeconds=0 disable the wait draining sat ahead of inFlightDataSize and inFlightUploads in S3ApiServer and shifted them off 8-byte alignment on 32-bit builds, so the atomic upload-limit counters panicked there. It now follows them. Helm: allInOne.s3.preStopSeconds inherits s3.preStopSeconds only when null; default treated an explicit 0 as unset. The s3 values and the grace period error now say a lifecycle preStop hook's time comes on top. Co-Authored-By: Claude Opus 5.5 * grace: add OnInterruptFirst for hooks that must run before any shutdown Interrupt hooks run in reverse registration order, so a hook cannot run ahead of a service that registers later, such as a volume server that is still loading its volumes. Co-Authored-By: Claude Opus 5.5 * s3: run the -preStopSeconds wait before every other interrupt hook Under weed server the volume server registers its hook once its volumes are loaded, often after the S3 gateway. Its hook then ran first and shut the volume server down, so the gateway spent the whole wait serving requests that could not read or write objects. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Chris Lu --- .github/workflows/helm_ci.yml | 43 +++++++ .../all-in-one/all-in-one-deployment.yaml | 13 ++- .../templates/filer/filer-statefulset.yaml | 9 +- .../seaweedfs/templates/s3/s3-deployment.yaml | 8 ++ k8s/charts/seaweedfs/values.yaml | 19 +++- weed/command/filer.go | 3 + weed/command/mini.go | 1 + weed/command/s3.go | 38 ++++++- weed/command/s3_shutdown_test.go | 106 ++++++++++++++++++ weed/command/server.go | 1 + weed/s3api/s3api_server.go | 1 + weed/s3api/s3api_status_handlers.go | 10 ++ weed/s3api/s3api_status_handlers_test.go | 23 ++++ weed/util/grace/signal_handling.go | 35 ++++-- .../grace/signal_handling_notsupported.go | 3 + weed/util/grace/signal_handling_test.go | 33 ++++++ 16 files changed, 331 insertions(+), 15 deletions(-) create mode 100644 weed/s3api/s3api_status_handlers_test.go create mode 100644 weed/util/grace/signal_handling_test.go diff --git a/.github/workflows/helm_ci.yml b/.github/workflows/helm_ci.yml index f3100db22d..aef8d2277c 100644 --- a/.github/workflows/helm_ci.yml +++ b/.github/workflows/helm_ci.yml @@ -104,8 +104,51 @@ jobs: grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "maxUnavailable: 0" grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "type: RollingUpdate" grep -A 5 "^ lifecycle:" /tmp/s3-rollout.yaml | grep -q -- "- sleep" + test "$(grep -c -- '-preStopSeconds' /tmp/s3-rollout-defaults.yaml)" -eq 0 echo "S3 rollout settings render correctly" + echo "=== Testing S3 preStopSeconds ===" + if helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \ + --set s3.enabled=true --set s3.preStopSeconds=20 > /tmp/s3-prestop-short.yaml 2>&1; then + echo "FAIL: s3.preStopSeconds=20 rendered with the default 10s grace period"; exit 1 + fi + grep -q "must be at least 35" /tmp/s3-prestop-short.yaml + helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \ + --set s3.enabled=true --set s3.preStopSeconds=20 \ + --set s3.terminationGracePeriodSeconds=35 > /tmp/s3-prestop.yaml + grep -q -- "-preStopSeconds=20 " /tmp/s3-prestop.yaml + grep -q "terminationGracePeriodSeconds: 35$" /tmp/s3-prestop.yaml + + helm template test $CHART_DIR --show-only templates/filer/filer-statefulset.yaml \ + --set filer.s3.enabled=true > /tmp/filer-s3-prestop-defaults.yaml + test "$(grep -c -- '-s3.preStopSeconds' /tmp/filer-s3-prestop-defaults.yaml)" -eq 0 + grep -q "terminationGracePeriodSeconds: 60$" /tmp/filer-s3-prestop-defaults.yaml + helm template test $CHART_DIR --show-only templates/filer/filer-statefulset.yaml \ + --set filer.s3.enabled=true --set filer.s3.preStopSeconds=20 > /tmp/filer-s3-prestop.yaml + grep -q -- "-s3.preStopSeconds=20 " /tmp/filer-s3-prestop.yaml + grep -q "terminationGracePeriodSeconds: 80$" /tmp/filer-s3-prestop.yaml + + helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \ + --set allInOne.enabled=true --set allInOne.s3.enabled=true > /tmp/aio-s3-prestop-defaults.yaml + test "$(grep -c -- '-s3.preStopSeconds' /tmp/aio-s3-prestop-defaults.yaml)" -eq 0 + grep -q "terminationGracePeriodSeconds: 60$" /tmp/aio-s3-prestop-defaults.yaml + helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \ + --set allInOne.enabled=true --set allInOne.s3.enabled=true \ + --set s3.preStopSeconds=20 > /tmp/aio-s3-prestop.yaml + grep -q -- "-s3.preStopSeconds=20 " /tmp/aio-s3-prestop.yaml + grep -q "terminationGracePeriodSeconds: 80$" /tmp/aio-s3-prestop.yaml + helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \ + --set allInOne.enabled=true --set allInOne.s3.enabled=true \ + --set s3.preStopSeconds=20 --set allInOne.s3.preStopSeconds=0 > /tmp/aio-s3-prestop-off.yaml + test "$(grep -c -- '-s3.preStopSeconds' /tmp/aio-s3-prestop-off.yaml)" -eq 0 + grep -q "terminationGracePeriodSeconds: 60$" /tmp/aio-s3-prestop-off.yaml + helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \ + --set allInOne.enabled=true --set allInOne.s3.enabled=true \ + --set s3.preStopSeconds=20 --set allInOne.s3.preStopSeconds=25 > /tmp/aio-s3-prestop-own.yaml + grep -q -- "-s3.preStopSeconds=25 " /tmp/aio-s3-prestop-own.yaml + grep -q "terminationGracePeriodSeconds: 85$" /tmp/aio-s3-prestop-own.yaml + echo "S3 preStopSeconds renders correctly" + echo "=== Testing the S3 config checksum is stable across renders ===" # s3-secret.yaml keeps its generated keys with lookup. Hashing the rendered # Secret drew new keys for the checksum whenever lookup found nothing (on diff --git a/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml b/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml index 1497bb4b45..e3bebb08a8 100644 --- a/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml @@ -1,5 +1,13 @@ {{- include "seaweedfs.compat" . -}} {{- if .Values.allInOne.enabled }} +{{- $s3PreStopSeconds := 0 }} +{{- if .Values.allInOne.s3.enabled }} +{{- $s3PreStopSeconds = .Values.allInOne.s3.preStopSeconds }} +{{- if kindIs "invalid" $s3PreStopSeconds }} +{{- $s3PreStopSeconds = .Values.s3.preStopSeconds }} +{{- end }} +{{- $s3PreStopSeconds = $s3PreStopSeconds | int }} +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: @@ -64,7 +72,7 @@ spec: {{- tpl .Values.allInOne.tolerations . | nindent 8 }} {{- end }} {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }} - terminationGracePeriodSeconds: 60 + terminationGracePeriodSeconds: {{ add 60 (max $s3PreStopSeconds 0) }} enableServiceLinks: false {{- if .Values.allInOne.priorityClassName }} priorityClassName: {{ .Values.allInOne.priorityClassName | quote }} @@ -260,6 +268,9 @@ spec: {{- if $auditLogConfig }} -s3.auditLogConfig=/etc/sw/s3/s3_auditLogConfig.json \ {{- end }} + {{- if gt $s3PreStopSeconds 0 }} + -s3.preStopSeconds={{ $s3PreStopSeconds }} \ + {{- end }} {{- end }} {{- if .Values.allInOne.sftp.enabled }} -sftp \ diff --git a/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml b/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml index d32da770cb..172ef1cf89 100644 --- a/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml @@ -1,5 +1,9 @@ {{- include "seaweedfs.compat" . -}} {{- if .Values.filer.enabled }} +{{- $s3PreStopSeconds := 0 }} +{{- if .Values.filer.s3.enabled }} +{{- $s3PreStopSeconds = .Values.filer.s3.preStopSeconds | int }} +{{- end }} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -69,7 +73,7 @@ spec: {{- end }} {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ .Values.filer.serviceAccountName | default (include "seaweedfs.serviceAccountName" .) | quote }} # for deleting statefulset pods after migration - terminationGracePeriodSeconds: 60 + terminationGracePeriodSeconds: {{ add 60 (max $s3PreStopSeconds 0) }} {{- if .Values.filer.priorityClassName }} priorityClassName: {{ .Values.filer.priorityClassName | quote }} {{- end }} @@ -278,6 +282,9 @@ spec: {{- if .Values.filer.s3.auditLogConfig }} -s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \ {{- end }} + {{- if gt $s3PreStopSeconds 0 }} + -s3.preStopSeconds={{ $s3PreStopSeconds }} \ + {{- end }} {{- end }} -master={{ include "seaweedfs.masterServerArg" . }} \ {{- range .Values.filer.extraArgs }} diff --git a/k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml b/k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml index c546dd5d40..4205ebac0e 100644 --- a/k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml @@ -1,5 +1,10 @@ {{- include "seaweedfs.compat" . -}} {{- if .Values.s3.enabled }} +{{- $preStopSeconds := .Values.s3.preStopSeconds | int }} +{{- $minGracePeriod := add $preStopSeconds 15 }} +{{- if and (gt $preStopSeconds 0) (lt (.Values.s3.terminationGracePeriodSeconds | int) $minGracePeriod) }} +{{- fail (printf "s3.terminationGracePeriodSeconds (%v) must be at least %d: the gateway keeps serving for s3.preStopSeconds (%d) after SIGTERM, then gives in-flight requests up to 15s to finish; add the time any s3.lifecycle.preStop hook takes" .Values.s3.terminationGracePeriodSeconds $minGracePeriod $preStopSeconds) -}} +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: @@ -154,6 +159,9 @@ spec: {{- if .Values.s3.auditLogConfig }} -auditLogConfig=/etc/sw/s3_auditLogConfig.json \ {{- end }} + {{- if gt $preStopSeconds 0 }} + -preStopSeconds={{ $preStopSeconds }} \ + {{- end }} -filer={{ include "seaweedfs.componentName" (list . "filer-client") }}.{{ include "seaweedfs.namespace" . }}:{{ .Values.filer.port }} \ {{- if .Values.s3.icebergPort }} -port.iceberg={{ .Values.s3.icebergPort }} \ diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 9f7e31be5f..493d6ef3da 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -1070,6 +1070,10 @@ filer: httpsPort: 0 # Suffix of the host name, {bucket}.{domainName} domainName: "" + # Seconds the gateway keeps serving after SIGTERM while its health checks + # return 503 (see s3.preStopSeconds). Added to the filer pod's 60s + # termination grace period. + preStopSeconds: 0 # enable user & permission to s3 (need to inject to all services) enableAuth: false # set to the name of an existing kubernetes Secret with the s3 json config file @@ -1118,8 +1122,17 @@ s3: # maxUnavailable: 0 # maxSurge: 25% updateStrategy: {} - # Time the s3 pod is given to shut down, including any preStop hook + # Time the s3 pod is given to shut down, including any preStop hook. + # With preStopSeconds set, it must be at least preStopSeconds + 15 (the + # in-flight request drain), or the chart refuses to render. A lifecycle + # preStop hook runs before SIGTERM, so add its duration on top; the chart + # cannot tell how long an exec hook takes. preStopSeconds replaces a + # preStop sleep hook. terminationGracePeriodSeconds: 10 + # Seconds the gateway keeps serving after SIGTERM while its health checks + # return 503, so the pod leaves the Service endpoints before it stops + # listening. 0 stops at once. + preStopSeconds: 0 bindAddress: 0.0.0.0 port: 8333 # add additional https port @@ -1318,7 +1331,8 @@ s3: # Lifecycle hooks for the s3 container. A preStop sleep lets the pod leave # the Service endpoints before it receives SIGTERM; keep it shorter than - # terminationGracePeriodSeconds. + # terminationGracePeriodSeconds. s3.preStopSeconds does the same from + # inside the gateway and also fails its health checks. # ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ # Example: # lifecycle: @@ -2011,6 +2025,7 @@ allInOne: # The s3-secret.yaml template only reads from .Values.s3.credentials. # See: s3.credentials.admin.accessKey, s3.credentials.read.accessKey auditLogConfig: null # S3 audit log configuration (null inherits from s3.auditLogConfig) + preStopSeconds: null # Seconds to keep serving after SIGTERM, added to the pod's 60s grace period (null inherits from s3.preStopSeconds, 0 disables) trafficDistribution: null # Service traffic distribution (e.g., "PreferClose"); auto-converts to "PreferSameZone" on k8s >=1.35 # You may specify buckets to be created during the install process. # Buckets may be exposed publicly by setting `anonymousRead` to `true` diff --git a/weed/command/filer.go b/weed/command/filer.go index cd4ac62454..8931abaafa 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -54,6 +54,8 @@ const allowUntrustedRemoteEndpointsUsage = "if true, a read of a remote-only ent // that runs the embedded IAM API. const iamSelfServiceRequiresGrantUsage = "if true, an identity needs an explicit iam: grant to manage its own access keys (CreateAccessKey, DeleteAccessKey, UpdateAccessKey, ListAccessKeys, GetUser), as on AWS. Default allows it to every authenticated identity." +const s3PreStopSecondsUsage = "seconds to keep serving after SIGTERM, with /status, /healthz and /readyz returning 503, before the S3 gateway shuts down. Gives load balancers and Kubernetes Services time to stop routing to it. 0 shuts down at once" + // allowBackslashInKeyUsage documents the S3 flag keeping '\' a literal // object-key character instead of normalizing it to '/'. Repeated '//' in a // key still collapses: a filer entry name cannot contain an empty segment. @@ -198,6 +200,7 @@ func init() { filerS3Options.allowUntrustedRemoteEndpoints = cmdFiler.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage) filerS3Options.allowBackslashInKey = cmdFiler.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage) filerS3Options.iamSelfServiceRequiresGrant = cmdFiler.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage) + filerS3Options.preStopSeconds = cmdFiler.Flag.Int("s3.preStopSeconds", 0, s3PreStopSecondsUsage) // start webdav on filer filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway") diff --git a/weed/command/mini.go b/weed/command/mini.go index f42db8827d..71304b20ff 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -572,6 +572,7 @@ func initMiniS3Flags() { miniS3Options.allowUntrustedRemoteEndpoints = cmdMini.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage) miniS3Options.allowBackslashInKey = cmdMini.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage) miniS3Options.iamSelfServiceRequiresGrant = cmdMini.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage) + miniS3Options.preStopSeconds = cmdMini.Flag.Int("s3.preStopSeconds", 0, s3PreStopSecondsUsage) // In mini mode, S3 uses the shared debug server started at line 681, not its own separate debug server miniS3Options.debug = new(bool) // explicitly false miniS3Options.debugPort = cmdMini.Flag.Int("s3.debug.port", 6060, "http port for debugging (unused in mini mode)") diff --git a/weed/command/s3.go b/weed/command/s3.go index 99c3506e78..8b70dc6623 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -10,6 +10,7 @@ import ( "runtime" "strconv" "strings" + "sync" "sync/atomic" "time" @@ -88,6 +89,7 @@ type S3Options struct { allowUntrustedRemoteEndpoints *bool allowBackslashInKey *bool iamSelfServiceRequiresGrant *bool + preStopSeconds *int // shutdownCtx, when non-nil, tells startS3Server to gracefully shut down // its HTTP/gRPC servers once the ctx is cancelled, in addition to on // interrupt. Used by weed mini to orchestrate an ordered shutdown. @@ -148,6 +150,7 @@ func init() { s3StandaloneOptions.allowUntrustedRemoteEndpoints = cmdS3.Flag.Bool("allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage) s3StandaloneOptions.allowBackslashInKey = cmdS3.Flag.Bool("allowBackslashInKey", false, allowBackslashInKeyUsage) s3StandaloneOptions.iamSelfServiceRequiresGrant = cmdS3.Flag.Bool("iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage) + s3StandaloneOptions.preStopSeconds = cmdS3.Flag.Int("preStopSeconds", 0, s3PreStopSecondsUsage) } var cmdS3 = &Command{ @@ -602,16 +605,47 @@ func (s3opt *S3Options) startS3Server() bool { func (s3opt *S3Options) newShutdown(stopGrpc func(), s3ApiServer *s3api.S3ApiServer, httpServers []*http.Server) func() { shutdown := newGracefulShutdown(stopGrpc, s3ApiServer.Shutdown, httpServers...) - grace.OnInterrupt(shutdown) + var cancelled <-chan struct{} if s3opt.shutdownCtx != nil { + cancelled = s3opt.shutdownCtx.Done() go func() { - <-s3opt.shutdownCtx.Done() + <-cancelled shutdown() }() } + preStop := sync.OnceFunc(preStopWait(s3ApiServer.StartDraining, s3opt.preStopDelay(), cancelled)) + // The wait has to finish before weed mini cancels shutdownCtx, and before + // weed server shuts down the filer and volume server, whose hooks may be + // registered after this one. + beforeMiniClientsShutdown(preStop) + grace.OnInterruptFirst(preStop) + grace.OnInterrupt(shutdown) return shutdown } +func (s3opt *S3Options) preStopDelay() time.Duration { + if s3opt.preStopSeconds == nil || *s3opt.preStopSeconds <= 0 { + return 0 + } + return time.Duration(*s3opt.preStopSeconds) * time.Second +} + +func preStopWait(startDraining func(), delay time.Duration, cancelled <-chan struct{}) func() { + return func() { + if delay <= 0 { + return + } + startDraining() + glog.V(0).Infof("S3 API Server failing health checks for %s before shutdown", delay) + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-timer.C: + case <-cancelled: + } + } +} + // vendsCredentials reports whether -iceberg.credentialRole is set, which // both catalogs vend through, whichever of them runs. func (s3opt *S3Options) vendsCredentials() bool { diff --git a/weed/command/s3_shutdown_test.go b/weed/command/s3_shutdown_test.go index 4b3126fd13..3c708e66ed 100644 --- a/weed/command/s3_shutdown_test.go +++ b/weed/command/s3_shutdown_test.go @@ -79,3 +79,109 @@ func TestS3ShutdownDrainsActiveRequestBeforeServeReturns(t *testing.T) { t.Fatal("Serve exit path did not return after the request drained") } } + +func TestS3PreStopRunsBeforeMiniCancelsClients(t *testing.T) { + savedClients, savedClusterCtx := miniClients, MiniClusterCtx + t.Cleanup(func() { miniClients, MiniClusterCtx = savedClients, savedClusterCtx }) + clusterCtx, stopCluster := context.WithCancel(context.Background()) + t.Cleanup(stopCluster) + MiniClusterCtx = clusterCtx + resetMiniClients() + + s3a := &s3api.S3ApiServer{} + mux := http.NewServeMux() + mux.HandleFunc("/healthz", s3a.StatusHandler) + mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + httpClosing := make(chan struct{}) + server.Config.RegisterOnShutdown(func() { close(httpClosing) }) + + get := func(path string) int { + t.Helper() + response, err := server.Client().Get(server.URL + path) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + return response.StatusCode + } + if code := get("/healthz"); code != http.StatusOK { + t.Fatalf("probe before shutdown returned %d", code) + } + + preStopSeconds := 3600 + s3opt := &S3Options{shutdownCtx: miniClientsCtx(), preStopSeconds: &preStopSeconds} + s3opt.newShutdown(func() {}, s3a, []*http.Server{server.Config}) + + // weed mini's interrupt hook, which runs before the S3 one. + triggerDone := make(chan struct{}) + go func() { triggerMiniClientsShutdown(5 * time.Second); close(triggerDone) }() + + deadline := time.Now().Add(5 * time.Second) + for get("/healthz") != http.StatusServiceUnavailable { + if time.Now().After(deadline) { + t.Fatal("probe did not start failing before mini cancelled its clients") + } + time.Sleep(5 * time.Millisecond) + } + if code := get("/bucket/key"); code != http.StatusNoContent { + t.Fatalf("request during the pre-stop wait returned %d", code) + } + select { + case <-httpClosing: + t.Fatal("HTTP drain started during the pre-stop wait") + default: + } + + stopCluster() + select { + case <-httpClosing: + case <-time.After(5 * time.Second): + t.Fatal("HTTP drain did not start once the wait ended") + } + select { + case <-triggerDone: + case <-time.After(5 * time.Second): + t.Fatal("mini client shutdown did not finish") + } +} + +func TestS3PreStopWaitZeroDelay(t *testing.T) { + drained := false + preStopWait(func() { drained = true }, 0, nil)() + if drained { + t.Error("probes failed with no pre-stop delay") + } +} + +func TestS3PreStopWaitElapses(t *testing.T) { + const delay = 50 * time.Millisecond + drained := false + started := time.Now() + preStopWait(func() { drained = true }, delay, nil)() + if !drained { + t.Error("probes did not fail during the pre-stop delay") + } + if elapsed := time.Since(started); elapsed < delay { + t.Fatalf("wait returned after %s, want at least %s", elapsed, delay) + } +} + +func TestS3PreStopDelay(t *testing.T) { + seconds := func(n int) *int { return &n } + for _, tc := range []struct { + name string + seconds *int + want time.Duration + }{ + {"unset", nil, 0}, + {"zero", seconds(0), 0}, + {"negative", seconds(-1), 0}, + {"set", seconds(20), 20 * time.Second}, + } { + if got := (&S3Options{preStopSeconds: tc.seconds}).preStopDelay(); got != tc.want { + t.Errorf("%s: preStopDelay() = %s, want %s", tc.name, got, tc.want) + } + } +} diff --git a/weed/command/server.go b/weed/command/server.go index e62378219d..d5ec4a34b7 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -212,6 +212,7 @@ func init() { s3Options.readerCacheSizeMB = cmdServer.Flag.Int64("s3.readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)") s3Options.allowUntrustedRemoteEndpoints = cmdServer.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage) s3Options.iamSelfServiceRequiresGrant = cmdServer.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage) + s3Options.preStopSeconds = cmdServer.Flag.Int("s3.preStopSeconds", 0, s3PreStopSecondsUsage) sftpOptions.port = cmdServer.Flag.Int("sftp.port", 2022, "SFTP server listen port") sftpOptions.idleTimeout = cmdServer.Flag.Int("sftp.idleTimeout", 10, "connection idle seconds") diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 0e8de408dc..e53ea0b6d1 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -115,6 +115,7 @@ type S3ApiServer struct { inFlightDataSize int64 inFlightUploads int64 inFlightDataLimitCond *sync.Cond + draining atomic.Bool embeddedIam *EmbeddedIamApi // Embedded IAM API server (when enabled) stsHandlers *STSHandlers // STS HTTP handlers for AssumeRoleWithWebIdentity // icebergCredentialRole is the role the Iceberg catalog assumes to vend diff --git a/weed/s3api/s3api_status_handlers.go b/weed/s3api/s3api_status_handlers.go index a70a883934..40ec4b52ac 100644 --- a/weed/s3api/s3api_status_handlers.go +++ b/weed/s3api/s3api_status_handlers.go @@ -13,6 +13,16 @@ func (s3a *S3ApiServer) StatusHandler(w http.ResponseWriter, r *http.Request) { s3err.WriteResponse(w, r, http.StatusOK, body, "application/json") return } + if s3a.draining.Load() { + s3err.WriteResponse(w, r, http.StatusServiceUnavailable, []byte{}, "") + return + } // write out the response code and content type header s3err.WriteResponse(w, r, http.StatusOK, []byte{}, "") } + +// StartDraining makes the health probes fail so load balancers stop routing +// new requests here, while requests that still arrive are served as usual. +func (s3a *S3ApiServer) StartDraining() { + s3a.draining.Store(true) +} diff --git a/weed/s3api/s3api_status_handlers_test.go b/weed/s3api/s3api_status_handlers_test.go new file mode 100644 index 0000000000..f0db7f95a3 --- /dev/null +++ b/weed/s3api/s3api_status_handlers_test.go @@ -0,0 +1,23 @@ +package s3api + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestStatusHandlerFailsWhileDraining(t *testing.T) { + s3a := &S3ApiServer{} + probe := func() int { + recorder := httptest.NewRecorder() + s3a.StatusHandler(recorder, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + return recorder.Code + } + if code := probe(); code != http.StatusOK { + t.Fatalf("probe returned %d before draining", code) + } + s3a.StartDraining() + if code := probe(); code != http.StatusServiceUnavailable { + t.Fatalf("probe returned %d while draining, want 503", code) + } +} diff --git a/weed/util/grace/signal_handling.go b/weed/util/grace/signal_handling.go index 75069fd5fc..08048667ae 100644 --- a/weed/util/grace/signal_handling.go +++ b/weed/util/grace/signal_handling.go @@ -15,6 +15,7 @@ import ( var signalChan chan os.Signal var interruptHooks = make([]func(), 0) +var firstInterruptHooks = make([]func(), 0) var interruptHookLock sync.RWMutex var reloadHooks = make([]func(), 0) var reloadHookLock sync.RWMutex @@ -43,21 +44,29 @@ func init() { } reloadHookLock.RUnlock() } else { - interruptHookLock.RLock() - // Execute hooks in reverse registration order (LIFO/defer-style) so - // later-started services shut down before the services they depend on. - for i := len(interruptHooks) - 1; i >= 0; i-- { - hook := interruptHooks[i] - glog.V(4).Infof("exec interrupt hook func name:%s", GetFunctionName(hook)) - hook() - } - interruptHookLock.RUnlock() + runInterruptHooks() os.Exit(0) } } }() } +// runInterruptHooks runs the OnInterruptFirst hooks in registration order, +// then the OnInterrupt hooks in reverse registration order (LIFO/defer-style) +// so later-started services shut down before the services they depend on. +func runInterruptHooks() { + interruptHookLock.RLock() + defer interruptHookLock.RUnlock() + for _, hook := range firstInterruptHooks { + hook() + } + for i := len(interruptHooks) - 1; i >= 0; i-- { + hook := interruptHooks[i] + glog.V(4).Infof("exec interrupt hook func name:%s", GetFunctionName(hook)) + hook() + } +} + func OnReload(fn func()) { // prevent reentry reloadHookLock.Lock() @@ -74,3 +83,11 @@ func OnInterrupt(fn func()) { // controlling terminal close, daemon not exit interruptHooks = append(interruptHooks, fn) } + +// OnInterruptFirst registers fn to run before every OnInterrupt hook, +// whenever those were registered. +func OnInterruptFirst(fn func()) { + interruptHookLock.Lock() + defer interruptHookLock.Unlock() + firstInterruptHooks = append(firstInterruptHooks, fn) +} diff --git a/weed/util/grace/signal_handling_notsupported.go b/weed/util/grace/signal_handling_notsupported.go index 7e57c1350b..a81a18e886 100644 --- a/weed/util/grace/signal_handling_notsupported.go +++ b/weed/util/grace/signal_handling_notsupported.go @@ -4,3 +4,6 @@ package grace func OnInterrupt(fn func()) { } + +func OnInterruptFirst(fn func()) { +} diff --git a/weed/util/grace/signal_handling_test.go b/weed/util/grace/signal_handling_test.go new file mode 100644 index 0000000000..1cedd545e5 --- /dev/null +++ b/weed/util/grace/signal_handling_test.go @@ -0,0 +1,33 @@ +//go:build !plan9 + +package grace + +import ( + "reflect" + "testing" +) + +func TestOnInterruptFirstRunsBeforeEarlierHooks(t *testing.T) { + interruptHookLock.Lock() + savedHooks, savedFirst := interruptHooks, firstInterruptHooks + interruptHooks, firstInterruptHooks = nil, nil + interruptHookLock.Unlock() + t.Cleanup(func() { + interruptHookLock.Lock() + interruptHooks, firstInterruptHooks = savedHooks, savedFirst + interruptHookLock.Unlock() + }) + + var order []string + OnInterrupt(func() { order = append(order, "volume") }) + OnInterruptFirst(func() { order = append(order, "s3 wait") }) + OnInterrupt(func() { order = append(order, "s3") }) + OnInterruptFirst(func() { order = append(order, "second wait") }) + + runInterruptHooks() + + want := []string{"s3 wait", "second wait", "s3", "volume"} + if !reflect.DeepEqual(order, want) { + t.Fatalf("hooks ran as %v, want %v", order, want) + } +}