diff --git a/.github/workflows/helm_ci.yml b/.github/workflows/helm_ci.yml index f3100db22d..aef8d2277c 100644 --- a/.github/workflows/helm_ci.yml +++ b/.github/workflows/helm_ci.yml @@ -104,8 +104,51 @@ jobs: grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "maxUnavailable: 0" grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "type: RollingUpdate" grep -A 5 "^ lifecycle:" /tmp/s3-rollout.yaml | grep -q -- "- sleep" + test "$(grep -c -- '-preStopSeconds' /tmp/s3-rollout-defaults.yaml)" -eq 0 echo "S3 rollout settings render correctly" + echo "=== Testing S3 preStopSeconds ===" + if helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \ + --set s3.enabled=true --set s3.preStopSeconds=20 > /tmp/s3-prestop-short.yaml 2>&1; then + echo "FAIL: s3.preStopSeconds=20 rendered with the default 10s grace period"; exit 1 + fi + grep -q "must be at least 35" /tmp/s3-prestop-short.yaml + helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \ + --set s3.enabled=true --set s3.preStopSeconds=20 \ + --set s3.terminationGracePeriodSeconds=35 > /tmp/s3-prestop.yaml + grep -q -- "-preStopSeconds=20 " /tmp/s3-prestop.yaml + grep -q "terminationGracePeriodSeconds: 35$" /tmp/s3-prestop.yaml + + helm template test $CHART_DIR --show-only templates/filer/filer-statefulset.yaml \ + --set filer.s3.enabled=true > /tmp/filer-s3-prestop-defaults.yaml + test "$(grep -c -- '-s3.preStopSeconds' /tmp/filer-s3-prestop-defaults.yaml)" -eq 0 + grep -q "terminationGracePeriodSeconds: 60$" /tmp/filer-s3-prestop-defaults.yaml + helm template test $CHART_DIR --show-only templates/filer/filer-statefulset.yaml \ + --set filer.s3.enabled=true --set filer.s3.preStopSeconds=20 > /tmp/filer-s3-prestop.yaml + grep -q -- "-s3.preStopSeconds=20 " /tmp/filer-s3-prestop.yaml + grep -q "terminationGracePeriodSeconds: 80$" /tmp/filer-s3-prestop.yaml + + helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \ + --set allInOne.enabled=true --set allInOne.s3.enabled=true > /tmp/aio-s3-prestop-defaults.yaml + test "$(grep -c -- '-s3.preStopSeconds' /tmp/aio-s3-prestop-defaults.yaml)" -eq 0 + grep -q "terminationGracePeriodSeconds: 60$" /tmp/aio-s3-prestop-defaults.yaml + helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \ + --set allInOne.enabled=true --set allInOne.s3.enabled=true \ + --set s3.preStopSeconds=20 > /tmp/aio-s3-prestop.yaml + grep -q -- "-s3.preStopSeconds=20 " /tmp/aio-s3-prestop.yaml + grep -q "terminationGracePeriodSeconds: 80$" /tmp/aio-s3-prestop.yaml + helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \ + --set allInOne.enabled=true --set allInOne.s3.enabled=true \ + --set s3.preStopSeconds=20 --set allInOne.s3.preStopSeconds=0 > /tmp/aio-s3-prestop-off.yaml + test "$(grep -c -- '-s3.preStopSeconds' /tmp/aio-s3-prestop-off.yaml)" -eq 0 + grep -q "terminationGracePeriodSeconds: 60$" /tmp/aio-s3-prestop-off.yaml + helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \ + --set allInOne.enabled=true --set allInOne.s3.enabled=true \ + --set s3.preStopSeconds=20 --set allInOne.s3.preStopSeconds=25 > /tmp/aio-s3-prestop-own.yaml + grep -q -- "-s3.preStopSeconds=25 " /tmp/aio-s3-prestop-own.yaml + grep -q "terminationGracePeriodSeconds: 85$" /tmp/aio-s3-prestop-own.yaml + echo "S3 preStopSeconds renders correctly" + echo "=== Testing the S3 config checksum is stable across renders ===" # s3-secret.yaml keeps its generated keys with lookup. Hashing the rendered # Secret drew new keys for the checksum whenever lookup found nothing (on diff --git a/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml b/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml index 1497bb4b45..e3bebb08a8 100644 --- a/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml @@ -1,5 +1,13 @@ {{- include "seaweedfs.compat" . -}} {{- if .Values.allInOne.enabled }} +{{- $s3PreStopSeconds := 0 }} +{{- if .Values.allInOne.s3.enabled }} +{{- $s3PreStopSeconds = .Values.allInOne.s3.preStopSeconds }} +{{- if kindIs "invalid" $s3PreStopSeconds }} +{{- $s3PreStopSeconds = .Values.s3.preStopSeconds }} +{{- end }} +{{- $s3PreStopSeconds = $s3PreStopSeconds | int }} +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: @@ -64,7 +72,7 @@ spec: {{- tpl .Values.allInOne.tolerations . | nindent 8 }} {{- end }} {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }} - terminationGracePeriodSeconds: 60 + terminationGracePeriodSeconds: {{ add 60 (max $s3PreStopSeconds 0) }} enableServiceLinks: false {{- if .Values.allInOne.priorityClassName }} priorityClassName: {{ .Values.allInOne.priorityClassName | quote }} @@ -260,6 +268,9 @@ spec: {{- if $auditLogConfig }} -s3.auditLogConfig=/etc/sw/s3/s3_auditLogConfig.json \ {{- end }} + {{- if gt $s3PreStopSeconds 0 }} + -s3.preStopSeconds={{ $s3PreStopSeconds }} \ + {{- end }} {{- end }} {{- if .Values.allInOne.sftp.enabled }} -sftp \ diff --git a/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml b/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml index d32da770cb..172ef1cf89 100644 --- a/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml @@ -1,5 +1,9 @@ {{- include "seaweedfs.compat" . -}} {{- if .Values.filer.enabled }} +{{- $s3PreStopSeconds := 0 }} +{{- if .Values.filer.s3.enabled }} +{{- $s3PreStopSeconds = .Values.filer.s3.preStopSeconds | int }} +{{- end }} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -69,7 +73,7 @@ spec: {{- end }} {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ .Values.filer.serviceAccountName | default (include "seaweedfs.serviceAccountName" .) | quote }} # for deleting statefulset pods after migration - terminationGracePeriodSeconds: 60 + terminationGracePeriodSeconds: {{ add 60 (max $s3PreStopSeconds 0) }} {{- if .Values.filer.priorityClassName }} priorityClassName: {{ .Values.filer.priorityClassName | quote }} {{- end }} @@ -278,6 +282,9 @@ spec: {{- if .Values.filer.s3.auditLogConfig }} -s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \ {{- end }} + {{- if gt $s3PreStopSeconds 0 }} + -s3.preStopSeconds={{ $s3PreStopSeconds }} \ + {{- end }} {{- end }} -master={{ include "seaweedfs.masterServerArg" . }} \ {{- range .Values.filer.extraArgs }} diff --git a/k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml b/k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml index c546dd5d40..4205ebac0e 100644 --- a/k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml @@ -1,5 +1,10 @@ {{- include "seaweedfs.compat" . -}} {{- if .Values.s3.enabled }} +{{- $preStopSeconds := .Values.s3.preStopSeconds | int }} +{{- $minGracePeriod := add $preStopSeconds 15 }} +{{- if and (gt $preStopSeconds 0) (lt (.Values.s3.terminationGracePeriodSeconds | int) $minGracePeriod) }} +{{- fail (printf "s3.terminationGracePeriodSeconds (%v) must be at least %d: the gateway keeps serving for s3.preStopSeconds (%d) after SIGTERM, then gives in-flight requests up to 15s to finish; add the time any s3.lifecycle.preStop hook takes" .Values.s3.terminationGracePeriodSeconds $minGracePeriod $preStopSeconds) -}} +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: @@ -154,6 +159,9 @@ spec: {{- if .Values.s3.auditLogConfig }} -auditLogConfig=/etc/sw/s3_auditLogConfig.json \ {{- end }} + {{- if gt $preStopSeconds 0 }} + -preStopSeconds={{ $preStopSeconds }} \ + {{- end }} -filer={{ include "seaweedfs.componentName" (list . "filer-client") }}.{{ include "seaweedfs.namespace" . }}:{{ .Values.filer.port }} \ {{- if .Values.s3.icebergPort }} -port.iceberg={{ .Values.s3.icebergPort }} \ diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 9f7e31be5f..493d6ef3da 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -1070,6 +1070,10 @@ filer: httpsPort: 0 # Suffix of the host name, {bucket}.{domainName} domainName: "" + # Seconds the gateway keeps serving after SIGTERM while its health checks + # return 503 (see s3.preStopSeconds). Added to the filer pod's 60s + # termination grace period. + preStopSeconds: 0 # enable user & permission to s3 (need to inject to all services) enableAuth: false # set to the name of an existing kubernetes Secret with the s3 json config file @@ -1118,8 +1122,17 @@ s3: # maxUnavailable: 0 # maxSurge: 25% updateStrategy: {} - # Time the s3 pod is given to shut down, including any preStop hook + # Time the s3 pod is given to shut down, including any preStop hook. + # With preStopSeconds set, it must be at least preStopSeconds + 15 (the + # in-flight request drain), or the chart refuses to render. A lifecycle + # preStop hook runs before SIGTERM, so add its duration on top; the chart + # cannot tell how long an exec hook takes. preStopSeconds replaces a + # preStop sleep hook. terminationGracePeriodSeconds: 10 + # Seconds the gateway keeps serving after SIGTERM while its health checks + # return 503, so the pod leaves the Service endpoints before it stops + # listening. 0 stops at once. + preStopSeconds: 0 bindAddress: 0.0.0.0 port: 8333 # add additional https port @@ -1318,7 +1331,8 @@ s3: # Lifecycle hooks for the s3 container. A preStop sleep lets the pod leave # the Service endpoints before it receives SIGTERM; keep it shorter than - # terminationGracePeriodSeconds. + # terminationGracePeriodSeconds. s3.preStopSeconds does the same from + # inside the gateway and also fails its health checks. # ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ # Example: # lifecycle: @@ -2011,6 +2025,7 @@ allInOne: # The s3-secret.yaml template only reads from .Values.s3.credentials. # See: s3.credentials.admin.accessKey, s3.credentials.read.accessKey auditLogConfig: null # S3 audit log configuration (null inherits from s3.auditLogConfig) + preStopSeconds: null # Seconds to keep serving after SIGTERM, added to the pod's 60s grace period (null inherits from s3.preStopSeconds, 0 disables) trafficDistribution: null # Service traffic distribution (e.g., "PreferClose"); auto-converts to "PreferSameZone" on k8s >=1.35 # You may specify buckets to be created during the install process. # Buckets may be exposed publicly by setting `anonymousRead` to `true` diff --git a/weed/command/filer.go b/weed/command/filer.go index cd4ac62454..8931abaafa 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -54,6 +54,8 @@ const allowUntrustedRemoteEndpointsUsage = "if true, a read of a remote-only ent // that runs the embedded IAM API. const iamSelfServiceRequiresGrantUsage = "if true, an identity needs an explicit iam: grant to manage its own access keys (CreateAccessKey, DeleteAccessKey, UpdateAccessKey, ListAccessKeys, GetUser), as on AWS. Default allows it to every authenticated identity." +const s3PreStopSecondsUsage = "seconds to keep serving after SIGTERM, with /status, /healthz and /readyz returning 503, before the S3 gateway shuts down. Gives load balancers and Kubernetes Services time to stop routing to it. 0 shuts down at once" + // allowBackslashInKeyUsage documents the S3 flag keeping '\' a literal // object-key character instead of normalizing it to '/'. Repeated '//' in a // key still collapses: a filer entry name cannot contain an empty segment. @@ -198,6 +200,7 @@ func init() { filerS3Options.allowUntrustedRemoteEndpoints = cmdFiler.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage) filerS3Options.allowBackslashInKey = cmdFiler.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage) filerS3Options.iamSelfServiceRequiresGrant = cmdFiler.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage) + filerS3Options.preStopSeconds = cmdFiler.Flag.Int("s3.preStopSeconds", 0, s3PreStopSecondsUsage) // start webdav on filer filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway") diff --git a/weed/command/mini.go b/weed/command/mini.go index f42db8827d..71304b20ff 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -572,6 +572,7 @@ func initMiniS3Flags() { miniS3Options.allowUntrustedRemoteEndpoints = cmdMini.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage) miniS3Options.allowBackslashInKey = cmdMini.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage) miniS3Options.iamSelfServiceRequiresGrant = cmdMini.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage) + miniS3Options.preStopSeconds = cmdMini.Flag.Int("s3.preStopSeconds", 0, s3PreStopSecondsUsage) // In mini mode, S3 uses the shared debug server started at line 681, not its own separate debug server miniS3Options.debug = new(bool) // explicitly false miniS3Options.debugPort = cmdMini.Flag.Int("s3.debug.port", 6060, "http port for debugging (unused in mini mode)") diff --git a/weed/command/s3.go b/weed/command/s3.go index 99c3506e78..8b70dc6623 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -10,6 +10,7 @@ import ( "runtime" "strconv" "strings" + "sync" "sync/atomic" "time" @@ -88,6 +89,7 @@ type S3Options struct { allowUntrustedRemoteEndpoints *bool allowBackslashInKey *bool iamSelfServiceRequiresGrant *bool + preStopSeconds *int // shutdownCtx, when non-nil, tells startS3Server to gracefully shut down // its HTTP/gRPC servers once the ctx is cancelled, in addition to on // interrupt. Used by weed mini to orchestrate an ordered shutdown. @@ -148,6 +150,7 @@ func init() { s3StandaloneOptions.allowUntrustedRemoteEndpoints = cmdS3.Flag.Bool("allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage) s3StandaloneOptions.allowBackslashInKey = cmdS3.Flag.Bool("allowBackslashInKey", false, allowBackslashInKeyUsage) s3StandaloneOptions.iamSelfServiceRequiresGrant = cmdS3.Flag.Bool("iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage) + s3StandaloneOptions.preStopSeconds = cmdS3.Flag.Int("preStopSeconds", 0, s3PreStopSecondsUsage) } var cmdS3 = &Command{ @@ -602,16 +605,47 @@ func (s3opt *S3Options) startS3Server() bool { func (s3opt *S3Options) newShutdown(stopGrpc func(), s3ApiServer *s3api.S3ApiServer, httpServers []*http.Server) func() { shutdown := newGracefulShutdown(stopGrpc, s3ApiServer.Shutdown, httpServers...) - grace.OnInterrupt(shutdown) + var cancelled <-chan struct{} if s3opt.shutdownCtx != nil { + cancelled = s3opt.shutdownCtx.Done() go func() { - <-s3opt.shutdownCtx.Done() + <-cancelled shutdown() }() } + preStop := sync.OnceFunc(preStopWait(s3ApiServer.StartDraining, s3opt.preStopDelay(), cancelled)) + // The wait has to finish before weed mini cancels shutdownCtx, and before + // weed server shuts down the filer and volume server, whose hooks may be + // registered after this one. + beforeMiniClientsShutdown(preStop) + grace.OnInterruptFirst(preStop) + grace.OnInterrupt(shutdown) return shutdown } +func (s3opt *S3Options) preStopDelay() time.Duration { + if s3opt.preStopSeconds == nil || *s3opt.preStopSeconds <= 0 { + return 0 + } + return time.Duration(*s3opt.preStopSeconds) * time.Second +} + +func preStopWait(startDraining func(), delay time.Duration, cancelled <-chan struct{}) func() { + return func() { + if delay <= 0 { + return + } + startDraining() + glog.V(0).Infof("S3 API Server failing health checks for %s before shutdown", delay) + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-timer.C: + case <-cancelled: + } + } +} + // vendsCredentials reports whether -iceberg.credentialRole is set, which // both catalogs vend through, whichever of them runs. func (s3opt *S3Options) vendsCredentials() bool { diff --git a/weed/command/s3_shutdown_test.go b/weed/command/s3_shutdown_test.go index 4b3126fd13..3c708e66ed 100644 --- a/weed/command/s3_shutdown_test.go +++ b/weed/command/s3_shutdown_test.go @@ -79,3 +79,109 @@ func TestS3ShutdownDrainsActiveRequestBeforeServeReturns(t *testing.T) { t.Fatal("Serve exit path did not return after the request drained") } } + +func TestS3PreStopRunsBeforeMiniCancelsClients(t *testing.T) { + savedClients, savedClusterCtx := miniClients, MiniClusterCtx + t.Cleanup(func() { miniClients, MiniClusterCtx = savedClients, savedClusterCtx }) + clusterCtx, stopCluster := context.WithCancel(context.Background()) + t.Cleanup(stopCluster) + MiniClusterCtx = clusterCtx + resetMiniClients() + + s3a := &s3api.S3ApiServer{} + mux := http.NewServeMux() + mux.HandleFunc("/healthz", s3a.StatusHandler) + mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + httpClosing := make(chan struct{}) + server.Config.RegisterOnShutdown(func() { close(httpClosing) }) + + get := func(path string) int { + t.Helper() + response, err := server.Client().Get(server.URL + path) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + return response.StatusCode + } + if code := get("/healthz"); code != http.StatusOK { + t.Fatalf("probe before shutdown returned %d", code) + } + + preStopSeconds := 3600 + s3opt := &S3Options{shutdownCtx: miniClientsCtx(), preStopSeconds: &preStopSeconds} + s3opt.newShutdown(func() {}, s3a, []*http.Server{server.Config}) + + // weed mini's interrupt hook, which runs before the S3 one. + triggerDone := make(chan struct{}) + go func() { triggerMiniClientsShutdown(5 * time.Second); close(triggerDone) }() + + deadline := time.Now().Add(5 * time.Second) + for get("/healthz") != http.StatusServiceUnavailable { + if time.Now().After(deadline) { + t.Fatal("probe did not start failing before mini cancelled its clients") + } + time.Sleep(5 * time.Millisecond) + } + if code := get("/bucket/key"); code != http.StatusNoContent { + t.Fatalf("request during the pre-stop wait returned %d", code) + } + select { + case <-httpClosing: + t.Fatal("HTTP drain started during the pre-stop wait") + default: + } + + stopCluster() + select { + case <-httpClosing: + case <-time.After(5 * time.Second): + t.Fatal("HTTP drain did not start once the wait ended") + } + select { + case <-triggerDone: + case <-time.After(5 * time.Second): + t.Fatal("mini client shutdown did not finish") + } +} + +func TestS3PreStopWaitZeroDelay(t *testing.T) { + drained := false + preStopWait(func() { drained = true }, 0, nil)() + if drained { + t.Error("probes failed with no pre-stop delay") + } +} + +func TestS3PreStopWaitElapses(t *testing.T) { + const delay = 50 * time.Millisecond + drained := false + started := time.Now() + preStopWait(func() { drained = true }, delay, nil)() + if !drained { + t.Error("probes did not fail during the pre-stop delay") + } + if elapsed := time.Since(started); elapsed < delay { + t.Fatalf("wait returned after %s, want at least %s", elapsed, delay) + } +} + +func TestS3PreStopDelay(t *testing.T) { + seconds := func(n int) *int { return &n } + for _, tc := range []struct { + name string + seconds *int + want time.Duration + }{ + {"unset", nil, 0}, + {"zero", seconds(0), 0}, + {"negative", seconds(-1), 0}, + {"set", seconds(20), 20 * time.Second}, + } { + if got := (&S3Options{preStopSeconds: tc.seconds}).preStopDelay(); got != tc.want { + t.Errorf("%s: preStopDelay() = %s, want %s", tc.name, got, tc.want) + } + } +} diff --git a/weed/command/server.go b/weed/command/server.go index e62378219d..d5ec4a34b7 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -212,6 +212,7 @@ func init() { s3Options.readerCacheSizeMB = cmdServer.Flag.Int64("s3.readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)") s3Options.allowUntrustedRemoteEndpoints = cmdServer.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage) s3Options.iamSelfServiceRequiresGrant = cmdServer.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage) + s3Options.preStopSeconds = cmdServer.Flag.Int("s3.preStopSeconds", 0, s3PreStopSecondsUsage) sftpOptions.port = cmdServer.Flag.Int("sftp.port", 2022, "SFTP server listen port") sftpOptions.idleTimeout = cmdServer.Flag.Int("sftp.idleTimeout", 10, "connection idle seconds") diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 0e8de408dc..e53ea0b6d1 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -115,6 +115,7 @@ type S3ApiServer struct { inFlightDataSize int64 inFlightUploads int64 inFlightDataLimitCond *sync.Cond + draining atomic.Bool embeddedIam *EmbeddedIamApi // Embedded IAM API server (when enabled) stsHandlers *STSHandlers // STS HTTP handlers for AssumeRoleWithWebIdentity // icebergCredentialRole is the role the Iceberg catalog assumes to vend diff --git a/weed/s3api/s3api_status_handlers.go b/weed/s3api/s3api_status_handlers.go index a70a883934..40ec4b52ac 100644 --- a/weed/s3api/s3api_status_handlers.go +++ b/weed/s3api/s3api_status_handlers.go @@ -13,6 +13,16 @@ func (s3a *S3ApiServer) StatusHandler(w http.ResponseWriter, r *http.Request) { s3err.WriteResponse(w, r, http.StatusOK, body, "application/json") return } + if s3a.draining.Load() { + s3err.WriteResponse(w, r, http.StatusServiceUnavailable, []byte{}, "") + return + } // write out the response code and content type header s3err.WriteResponse(w, r, http.StatusOK, []byte{}, "") } + +// StartDraining makes the health probes fail so load balancers stop routing +// new requests here, while requests that still arrive are served as usual. +func (s3a *S3ApiServer) StartDraining() { + s3a.draining.Store(true) +} diff --git a/weed/s3api/s3api_status_handlers_test.go b/weed/s3api/s3api_status_handlers_test.go new file mode 100644 index 0000000000..f0db7f95a3 --- /dev/null +++ b/weed/s3api/s3api_status_handlers_test.go @@ -0,0 +1,23 @@ +package s3api + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestStatusHandlerFailsWhileDraining(t *testing.T) { + s3a := &S3ApiServer{} + probe := func() int { + recorder := httptest.NewRecorder() + s3a.StatusHandler(recorder, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + return recorder.Code + } + if code := probe(); code != http.StatusOK { + t.Fatalf("probe returned %d before draining", code) + } + s3a.StartDraining() + if code := probe(); code != http.StatusServiceUnavailable { + t.Fatalf("probe returned %d while draining, want 503", code) + } +} diff --git a/weed/util/grace/signal_handling.go b/weed/util/grace/signal_handling.go index 75069fd5fc..08048667ae 100644 --- a/weed/util/grace/signal_handling.go +++ b/weed/util/grace/signal_handling.go @@ -15,6 +15,7 @@ import ( var signalChan chan os.Signal var interruptHooks = make([]func(), 0) +var firstInterruptHooks = make([]func(), 0) var interruptHookLock sync.RWMutex var reloadHooks = make([]func(), 0) var reloadHookLock sync.RWMutex @@ -43,21 +44,29 @@ func init() { } reloadHookLock.RUnlock() } else { - interruptHookLock.RLock() - // Execute hooks in reverse registration order (LIFO/defer-style) so - // later-started services shut down before the services they depend on. - for i := len(interruptHooks) - 1; i >= 0; i-- { - hook := interruptHooks[i] - glog.V(4).Infof("exec interrupt hook func name:%s", GetFunctionName(hook)) - hook() - } - interruptHookLock.RUnlock() + runInterruptHooks() os.Exit(0) } } }() } +// runInterruptHooks runs the OnInterruptFirst hooks in registration order, +// then the OnInterrupt hooks in reverse registration order (LIFO/defer-style) +// so later-started services shut down before the services they depend on. +func runInterruptHooks() { + interruptHookLock.RLock() + defer interruptHookLock.RUnlock() + for _, hook := range firstInterruptHooks { + hook() + } + for i := len(interruptHooks) - 1; i >= 0; i-- { + hook := interruptHooks[i] + glog.V(4).Infof("exec interrupt hook func name:%s", GetFunctionName(hook)) + hook() + } +} + func OnReload(fn func()) { // prevent reentry reloadHookLock.Lock() @@ -74,3 +83,11 @@ func OnInterrupt(fn func()) { // controlling terminal close, daemon not exit interruptHooks = append(interruptHooks, fn) } + +// OnInterruptFirst registers fn to run before every OnInterrupt hook, +// whenever those were registered. +func OnInterruptFirst(fn func()) { + interruptHookLock.Lock() + defer interruptHookLock.Unlock() + firstInterruptHooks = append(firstInterruptHooks, fn) +} diff --git a/weed/util/grace/signal_handling_notsupported.go b/weed/util/grace/signal_handling_notsupported.go index 7e57c1350b..a81a18e886 100644 --- a/weed/util/grace/signal_handling_notsupported.go +++ b/weed/util/grace/signal_handling_notsupported.go @@ -4,3 +4,6 @@ package grace func OnInterrupt(fn func()) { } + +func OnInterruptFirst(fn func()) { +} diff --git a/weed/util/grace/signal_handling_test.go b/weed/util/grace/signal_handling_test.go new file mode 100644 index 0000000000..1cedd545e5 --- /dev/null +++ b/weed/util/grace/signal_handling_test.go @@ -0,0 +1,33 @@ +//go:build !plan9 + +package grace + +import ( + "reflect" + "testing" +) + +func TestOnInterruptFirstRunsBeforeEarlierHooks(t *testing.T) { + interruptHookLock.Lock() + savedHooks, savedFirst := interruptHooks, firstInterruptHooks + interruptHooks, firstInterruptHooks = nil, nil + interruptHookLock.Unlock() + t.Cleanup(func() { + interruptHookLock.Lock() + interruptHooks, firstInterruptHooks = savedHooks, savedFirst + interruptHookLock.Unlock() + }) + + var order []string + OnInterrupt(func() { order = append(order, "volume") }) + OnInterruptFirst(func() { order = append(order, "s3 wait") }) + OnInterrupt(func() { order = append(order, "s3") }) + OnInterruptFirst(func() { order = append(order, "second wait") }) + + runInterruptHooks() + + want := []string{"s3 wait", "second wait", "s3", "volume"} + if !reflect.DeepEqual(order, want) { + t.Fatalf("hooks ran as %v, want %v", order, want) + } +}