mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 09:46:24 +01:00
s3: keep serving for -preStopSeconds after SIGTERM (#12703)
* s3: keep serving for -preStopSeconds after SIGTERM On SIGTERM the gateway closed its listeners at once, so requests a load balancer or Kubernetes Service still routed to it failed until the endpoint was removed. With -preStopSeconds set, the gateway makes /status, /healthz and /readyz return 503, keeps serving for that many seconds, then runs the existing graceful shutdown. 0 (the default) keeps the old behavior. The flag is registered for weed s3 and as -s3.preStopSeconds for weed filer, weed server and weed mini. A cancelled weed mini cluster context ends the wait early. The Helm chart passes s3.preStopSeconds and filer.s3.preStopSeconds through. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3: assert the preStopSeconds chart render under set -e, drop timing from the probe test Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3: wait before weed mini cancels its clients; size grace periods in the chart weed mini's interrupt hook runs before the S3 one and cancels the S3 gateway's shutdownCtx, so the pre-stop wait never happened there. The wait is now also registered with beforeMiniClientsShutdown, which runs before that cancel, and a sync.OnceFunc keeps the interrupt hook from waiting a second time. Outside weed mini the registration is a no-op. Helm: the s3 deployment fails to render when terminationGracePeriodSeconds is below preStopSeconds + 15. The filer statefulset and the all-in-one deployment, whose grace period is fixed at 60s, add preStopSeconds to it. allInOne.s3.preStopSeconds is passed through, inheriting s3.preStopSeconds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3: keep 64-bit counters aligned on 386; let allInOne.s3.preStopSeconds=0 disable the wait draining sat ahead of inFlightDataSize and inFlightUploads in S3ApiServer and shifted them off 8-byte alignment on 32-bit builds, so the atomic upload-limit counters panicked there. It now follows them. Helm: allInOne.s3.preStopSeconds inherits s3.preStopSeconds only when null; default treated an explicit 0 as unset. The s3 values and the grace period error now say a lifecycle preStop hook's time comes on top. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * grace: add OnInterruptFirst for hooks that must run before any shutdown Interrupt hooks run in reverse registration order, so a hook cannot run ahead of a service that registers later, such as a volume server that is still loading its volumes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * s3: run the -preStopSeconds wait before every other interrupt hook Under weed server the volume server registers its hook once its volumes are loaded, often after the S3 gateway. Its hook then ran first and shut the volume server down, so the gateway spent the whole wait serving requests that could not read or write objects. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
This commit is contained in:
16 files changed
+331
-15
No files matched your search
@@ -104,8 +104,51 @@ jobs:
|
||||
grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "maxUnavailable: 0"
|
||||
grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "type: RollingUpdate"
|
||||
grep -A 5 "^ lifecycle:" /tmp/s3-rollout.yaml | grep -q -- "- sleep"
|
||||
test "$(grep -c -- '-preStopSeconds' /tmp/s3-rollout-defaults.yaml)" -eq 0
|
||||
echo "S3 rollout settings render correctly"
|
||||
|
||||
echo "=== Testing S3 preStopSeconds ==="
|
||||
if helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \
|
||||
--set s3.enabled=true --set s3.preStopSeconds=20 > /tmp/s3-prestop-short.yaml 2>&1; then
|
||||
echo "FAIL: s3.preStopSeconds=20 rendered with the default 10s grace period"; exit 1
|
||||
fi
|
||||
grep -q "must be at least 35" /tmp/s3-prestop-short.yaml
|
||||
helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \
|
||||
--set s3.enabled=true --set s3.preStopSeconds=20 \
|
||||
--set s3.terminationGracePeriodSeconds=35 > /tmp/s3-prestop.yaml
|
||||
grep -q -- "-preStopSeconds=20 " /tmp/s3-prestop.yaml
|
||||
grep -q "terminationGracePeriodSeconds: 35$" /tmp/s3-prestop.yaml
|
||||
|
||||
helm template test $CHART_DIR --show-only templates/filer/filer-statefulset.yaml \
|
||||
--set filer.s3.enabled=true > /tmp/filer-s3-prestop-defaults.yaml
|
||||
test "$(grep -c -- '-s3.preStopSeconds' /tmp/filer-s3-prestop-defaults.yaml)" -eq 0
|
||||
grep -q "terminationGracePeriodSeconds: 60$" /tmp/filer-s3-prestop-defaults.yaml
|
||||
helm template test $CHART_DIR --show-only templates/filer/filer-statefulset.yaml \
|
||||
--set filer.s3.enabled=true --set filer.s3.preStopSeconds=20 > /tmp/filer-s3-prestop.yaml
|
||||
grep -q -- "-s3.preStopSeconds=20 " /tmp/filer-s3-prestop.yaml
|
||||
grep -q "terminationGracePeriodSeconds: 80$" /tmp/filer-s3-prestop.yaml
|
||||
|
||||
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
|
||||
--set allInOne.enabled=true --set allInOne.s3.enabled=true > /tmp/aio-s3-prestop-defaults.yaml
|
||||
test "$(grep -c -- '-s3.preStopSeconds' /tmp/aio-s3-prestop-defaults.yaml)" -eq 0
|
||||
grep -q "terminationGracePeriodSeconds: 60$" /tmp/aio-s3-prestop-defaults.yaml
|
||||
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
|
||||
--set allInOne.enabled=true --set allInOne.s3.enabled=true \
|
||||
--set s3.preStopSeconds=20 > /tmp/aio-s3-prestop.yaml
|
||||
grep -q -- "-s3.preStopSeconds=20 " /tmp/aio-s3-prestop.yaml
|
||||
grep -q "terminationGracePeriodSeconds: 80$" /tmp/aio-s3-prestop.yaml
|
||||
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
|
||||
--set allInOne.enabled=true --set allInOne.s3.enabled=true \
|
||||
--set s3.preStopSeconds=20 --set allInOne.s3.preStopSeconds=0 > /tmp/aio-s3-prestop-off.yaml
|
||||
test "$(grep -c -- '-s3.preStopSeconds' /tmp/aio-s3-prestop-off.yaml)" -eq 0
|
||||
grep -q "terminationGracePeriodSeconds: 60$" /tmp/aio-s3-prestop-off.yaml
|
||||
helm template test $CHART_DIR --show-only templates/all-in-one/all-in-one-deployment.yaml \
|
||||
--set allInOne.enabled=true --set allInOne.s3.enabled=true \
|
||||
--set s3.preStopSeconds=20 --set allInOne.s3.preStopSeconds=25 > /tmp/aio-s3-prestop-own.yaml
|
||||
grep -q -- "-s3.preStopSeconds=25 " /tmp/aio-s3-prestop-own.yaml
|
||||
grep -q "terminationGracePeriodSeconds: 85$" /tmp/aio-s3-prestop-own.yaml
|
||||
echo "S3 preStopSeconds renders correctly"
|
||||
|
||||
echo "=== Testing the S3 config checksum is stable across renders ==="
|
||||
# s3-secret.yaml keeps its generated keys with lookup. Hashing the rendered
|
||||
# Secret drew new keys for the checksum whenever lookup found nothing (on
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if .Values.allInOne.enabled }}
|
||||
{{- $s3PreStopSeconds := 0 }}
|
||||
{{- if .Values.allInOne.s3.enabled }}
|
||||
{{- $s3PreStopSeconds = .Values.allInOne.s3.preStopSeconds }}
|
||||
{{- if kindIs "invalid" $s3PreStopSeconds }}
|
||||
{{- $s3PreStopSeconds = .Values.s3.preStopSeconds }}
|
||||
{{- end }}
|
||||
{{- $s3PreStopSeconds = $s3PreStopSeconds | int }}
|
||||
{{- end }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -64,7 +72,7 @@ spec:
|
||||
{{- tpl .Values.allInOne.tolerations . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
|
||||
terminationGracePeriodSeconds: 60
|
||||
terminationGracePeriodSeconds: {{ add 60 (max $s3PreStopSeconds 0) }}
|
||||
enableServiceLinks: false
|
||||
{{- if .Values.allInOne.priorityClassName }}
|
||||
priorityClassName: {{ .Values.allInOne.priorityClassName | quote }}
|
||||
@@ -260,6 +268,9 @@ spec:
|
||||
{{- if $auditLogConfig }}
|
||||
-s3.auditLogConfig=/etc/sw/s3/s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
{{- if gt $s3PreStopSeconds 0 }}
|
||||
-s3.preStopSeconds={{ $s3PreStopSeconds }} \
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.allInOne.sftp.enabled }}
|
||||
-sftp \
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if .Values.filer.enabled }}
|
||||
{{- $s3PreStopSeconds := 0 }}
|
||||
{{- if .Values.filer.s3.enabled }}
|
||||
{{- $s3PreStopSeconds = .Values.filer.s3.preStopSeconds | int }}
|
||||
{{- end }}
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
@@ -69,7 +73,7 @@ spec:
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
|
||||
serviceAccountName: {{ .Values.filer.serviceAccountName | default (include "seaweedfs.serviceAccountName" .) | quote }} # for deleting statefulset pods after migration
|
||||
terminationGracePeriodSeconds: 60
|
||||
terminationGracePeriodSeconds: {{ add 60 (max $s3PreStopSeconds 0) }}
|
||||
{{- if .Values.filer.priorityClassName }}
|
||||
priorityClassName: {{ .Values.filer.priorityClassName | quote }}
|
||||
{{- end }}
|
||||
@@ -278,6 +282,9 @@ spec:
|
||||
{{- if .Values.filer.s3.auditLogConfig }}
|
||||
-s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
{{- if gt $s3PreStopSeconds 0 }}
|
||||
-s3.preStopSeconds={{ $s3PreStopSeconds }} \
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
-master={{ include "seaweedfs.masterServerArg" . }} \
|
||||
{{- range .Values.filer.extraArgs }}
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if .Values.s3.enabled }}
|
||||
{{- $preStopSeconds := .Values.s3.preStopSeconds | int }}
|
||||
{{- $minGracePeriod := add $preStopSeconds 15 }}
|
||||
{{- if and (gt $preStopSeconds 0) (lt (.Values.s3.terminationGracePeriodSeconds | int) $minGracePeriod) }}
|
||||
{{- fail (printf "s3.terminationGracePeriodSeconds (%v) must be at least %d: the gateway keeps serving for s3.preStopSeconds (%d) after SIGTERM, then gives in-flight requests up to 15s to finish; add the time any s3.lifecycle.preStop hook takes" .Values.s3.terminationGracePeriodSeconds $minGracePeriod $preStopSeconds) -}}
|
||||
{{- end }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -154,6 +159,9 @@ spec:
|
||||
{{- if .Values.s3.auditLogConfig }}
|
||||
-auditLogConfig=/etc/sw/s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
{{- if gt $preStopSeconds 0 }}
|
||||
-preStopSeconds={{ $preStopSeconds }} \
|
||||
{{- end }}
|
||||
-filer={{ include "seaweedfs.componentName" (list . "filer-client") }}.{{ include "seaweedfs.namespace" . }}:{{ .Values.filer.port }} \
|
||||
{{- if .Values.s3.icebergPort }}
|
||||
-port.iceberg={{ .Values.s3.icebergPort }} \
|
||||
|
||||
@@ -1070,6 +1070,10 @@ filer:
|
||||
httpsPort: 0
|
||||
# Suffix of the host name, {bucket}.{domainName}
|
||||
domainName: ""
|
||||
# Seconds the gateway keeps serving after SIGTERM while its health checks
|
||||
# return 503 (see s3.preStopSeconds). Added to the filer pod's 60s
|
||||
# termination grace period.
|
||||
preStopSeconds: 0
|
||||
# enable user & permission to s3 (need to inject to all services)
|
||||
enableAuth: false
|
||||
# set to the name of an existing kubernetes Secret with the s3 json config file
|
||||
@@ -1118,8 +1122,17 @@ s3:
|
||||
# maxUnavailable: 0
|
||||
# maxSurge: 25%
|
||||
updateStrategy: {}
|
||||
# Time the s3 pod is given to shut down, including any preStop hook
|
||||
# Time the s3 pod is given to shut down, including any preStop hook.
|
||||
# With preStopSeconds set, it must be at least preStopSeconds + 15 (the
|
||||
# in-flight request drain), or the chart refuses to render. A lifecycle
|
||||
# preStop hook runs before SIGTERM, so add its duration on top; the chart
|
||||
# cannot tell how long an exec hook takes. preStopSeconds replaces a
|
||||
# preStop sleep hook.
|
||||
terminationGracePeriodSeconds: 10
|
||||
# Seconds the gateway keeps serving after SIGTERM while its health checks
|
||||
# return 503, so the pod leaves the Service endpoints before it stops
|
||||
# listening. 0 stops at once.
|
||||
preStopSeconds: 0
|
||||
bindAddress: 0.0.0.0
|
||||
port: 8333
|
||||
# add additional https port
|
||||
@@ -1318,7 +1331,8 @@ s3:
|
||||
|
||||
# Lifecycle hooks for the s3 container. A preStop sleep lets the pod leave
|
||||
# the Service endpoints before it receives SIGTERM; keep it shorter than
|
||||
# terminationGracePeriodSeconds.
|
||||
# terminationGracePeriodSeconds. s3.preStopSeconds does the same from
|
||||
# inside the gateway and also fails its health checks.
|
||||
# ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
|
||||
# Example:
|
||||
# lifecycle:
|
||||
@@ -2011,6 +2025,7 @@ allInOne:
|
||||
# The s3-secret.yaml template only reads from .Values.s3.credentials.
|
||||
# See: s3.credentials.admin.accessKey, s3.credentials.read.accessKey
|
||||
auditLogConfig: null # S3 audit log configuration (null inherits from s3.auditLogConfig)
|
||||
preStopSeconds: null # Seconds to keep serving after SIGTERM, added to the pod's 60s grace period (null inherits from s3.preStopSeconds, 0 disables)
|
||||
trafficDistribution: null # Service traffic distribution (e.g., "PreferClose"); auto-converts to "PreferSameZone" on k8s >=1.35
|
||||
# You may specify buckets to be created during the install process.
|
||||
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
|
||||
|
||||
@@ -54,6 +54,8 @@ const allowUntrustedRemoteEndpointsUsage = "if true, a read of a remote-only ent
|
||||
// that runs the embedded IAM API.
|
||||
const iamSelfServiceRequiresGrantUsage = "if true, an identity needs an explicit iam: grant to manage its own access keys (CreateAccessKey, DeleteAccessKey, UpdateAccessKey, ListAccessKeys, GetUser), as on AWS. Default allows it to every authenticated identity."
|
||||
|
||||
const s3PreStopSecondsUsage = "seconds to keep serving after SIGTERM, with /status, /healthz and /readyz returning 503, before the S3 gateway shuts down. Gives load balancers and Kubernetes Services time to stop routing to it. 0 shuts down at once"
|
||||
|
||||
// allowBackslashInKeyUsage documents the S3 flag keeping '\' a literal
|
||||
// object-key character instead of normalizing it to '/'. Repeated '//' in a
|
||||
// key still collapses: a filer entry name cannot contain an empty segment.
|
||||
@@ -198,6 +200,7 @@ func init() {
|
||||
filerS3Options.allowUntrustedRemoteEndpoints = cmdFiler.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
|
||||
filerS3Options.allowBackslashInKey = cmdFiler.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage)
|
||||
filerS3Options.iamSelfServiceRequiresGrant = cmdFiler.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
|
||||
filerS3Options.preStopSeconds = cmdFiler.Flag.Int("s3.preStopSeconds", 0, s3PreStopSecondsUsage)
|
||||
|
||||
// start webdav on filer
|
||||
filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
|
||||
|
||||
@@ -572,6 +572,7 @@ func initMiniS3Flags() {
|
||||
miniS3Options.allowUntrustedRemoteEndpoints = cmdMini.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
|
||||
miniS3Options.allowBackslashInKey = cmdMini.Flag.Bool("s3.allowBackslashInKey", false, allowBackslashInKeyUsage)
|
||||
miniS3Options.iamSelfServiceRequiresGrant = cmdMini.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
|
||||
miniS3Options.preStopSeconds = cmdMini.Flag.Int("s3.preStopSeconds", 0, s3PreStopSecondsUsage)
|
||||
// In mini mode, S3 uses the shared debug server started at line 681, not its own separate debug server
|
||||
miniS3Options.debug = new(bool) // explicitly false
|
||||
miniS3Options.debugPort = cmdMini.Flag.Int("s3.debug.port", 6060, "http port for debugging (unused in mini mode)")
|
||||
|
||||
+36
-2
@@ -10,6 +10,7 @@ import (
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -88,6 +89,7 @@ type S3Options struct {
|
||||
allowUntrustedRemoteEndpoints *bool
|
||||
allowBackslashInKey *bool
|
||||
iamSelfServiceRequiresGrant *bool
|
||||
preStopSeconds *int
|
||||
// shutdownCtx, when non-nil, tells startS3Server to gracefully shut down
|
||||
// its HTTP/gRPC servers once the ctx is cancelled, in addition to on
|
||||
// interrupt. Used by weed mini to orchestrate an ordered shutdown.
|
||||
@@ -148,6 +150,7 @@ func init() {
|
||||
s3StandaloneOptions.allowUntrustedRemoteEndpoints = cmdS3.Flag.Bool("allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
|
||||
s3StandaloneOptions.allowBackslashInKey = cmdS3.Flag.Bool("allowBackslashInKey", false, allowBackslashInKeyUsage)
|
||||
s3StandaloneOptions.iamSelfServiceRequiresGrant = cmdS3.Flag.Bool("iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
|
||||
s3StandaloneOptions.preStopSeconds = cmdS3.Flag.Int("preStopSeconds", 0, s3PreStopSecondsUsage)
|
||||
}
|
||||
|
||||
var cmdS3 = &Command{
|
||||
@@ -602,16 +605,47 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
|
||||
func (s3opt *S3Options) newShutdown(stopGrpc func(), s3ApiServer *s3api.S3ApiServer, httpServers []*http.Server) func() {
|
||||
shutdown := newGracefulShutdown(stopGrpc, s3ApiServer.Shutdown, httpServers...)
|
||||
grace.OnInterrupt(shutdown)
|
||||
var cancelled <-chan struct{}
|
||||
if s3opt.shutdownCtx != nil {
|
||||
cancelled = s3opt.shutdownCtx.Done()
|
||||
go func() {
|
||||
<-s3opt.shutdownCtx.Done()
|
||||
<-cancelled
|
||||
shutdown()
|
||||
}()
|
||||
}
|
||||
preStop := sync.OnceFunc(preStopWait(s3ApiServer.StartDraining, s3opt.preStopDelay(), cancelled))
|
||||
// The wait has to finish before weed mini cancels shutdownCtx, and before
|
||||
// weed server shuts down the filer and volume server, whose hooks may be
|
||||
// registered after this one.
|
||||
beforeMiniClientsShutdown(preStop)
|
||||
grace.OnInterruptFirst(preStop)
|
||||
grace.OnInterrupt(shutdown)
|
||||
return shutdown
|
||||
}
|
||||
|
||||
func (s3opt *S3Options) preStopDelay() time.Duration {
|
||||
if s3opt.preStopSeconds == nil || *s3opt.preStopSeconds <= 0 {
|
||||
return 0
|
||||
}
|
||||
return time.Duration(*s3opt.preStopSeconds) * time.Second
|
||||
}
|
||||
|
||||
func preStopWait(startDraining func(), delay time.Duration, cancelled <-chan struct{}) func() {
|
||||
return func() {
|
||||
if delay <= 0 {
|
||||
return
|
||||
}
|
||||
startDraining()
|
||||
glog.V(0).Infof("S3 API Server failing health checks for %s before shutdown", delay)
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
case <-cancelled:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// vendsCredentials reports whether -iceberg.credentialRole is set, which
|
||||
// both catalogs vend through, whichever of them runs.
|
||||
func (s3opt *S3Options) vendsCredentials() bool {
|
||||
|
||||
@@ -79,3 +79,109 @@ func TestS3ShutdownDrainsActiveRequestBeforeServeReturns(t *testing.T) {
|
||||
t.Fatal("Serve exit path did not return after the request drained")
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3PreStopRunsBeforeMiniCancelsClients(t *testing.T) {
|
||||
savedClients, savedClusterCtx := miniClients, MiniClusterCtx
|
||||
t.Cleanup(func() { miniClients, MiniClusterCtx = savedClients, savedClusterCtx })
|
||||
clusterCtx, stopCluster := context.WithCancel(context.Background())
|
||||
t.Cleanup(stopCluster)
|
||||
MiniClusterCtx = clusterCtx
|
||||
resetMiniClients()
|
||||
|
||||
s3a := &s3api.S3ApiServer{}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/healthz", s3a.StatusHandler)
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })
|
||||
server := httptest.NewServer(mux)
|
||||
t.Cleanup(server.Close)
|
||||
httpClosing := make(chan struct{})
|
||||
server.Config.RegisterOnShutdown(func() { close(httpClosing) })
|
||||
|
||||
get := func(path string) int {
|
||||
t.Helper()
|
||||
response, err := server.Client().Get(server.URL + path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response.Body.Close()
|
||||
return response.StatusCode
|
||||
}
|
||||
if code := get("/healthz"); code != http.StatusOK {
|
||||
t.Fatalf("probe before shutdown returned %d", code)
|
||||
}
|
||||
|
||||
preStopSeconds := 3600
|
||||
s3opt := &S3Options{shutdownCtx: miniClientsCtx(), preStopSeconds: &preStopSeconds}
|
||||
s3opt.newShutdown(func() {}, s3a, []*http.Server{server.Config})
|
||||
|
||||
// weed mini's interrupt hook, which runs before the S3 one.
|
||||
triggerDone := make(chan struct{})
|
||||
go func() { triggerMiniClientsShutdown(5 * time.Second); close(triggerDone) }()
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for get("/healthz") != http.StatusServiceUnavailable {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("probe did not start failing before mini cancelled its clients")
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if code := get("/bucket/key"); code != http.StatusNoContent {
|
||||
t.Fatalf("request during the pre-stop wait returned %d", code)
|
||||
}
|
||||
select {
|
||||
case <-httpClosing:
|
||||
t.Fatal("HTTP drain started during the pre-stop wait")
|
||||
default:
|
||||
}
|
||||
|
||||
stopCluster()
|
||||
select {
|
||||
case <-httpClosing:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("HTTP drain did not start once the wait ended")
|
||||
}
|
||||
select {
|
||||
case <-triggerDone:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("mini client shutdown did not finish")
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3PreStopWaitZeroDelay(t *testing.T) {
|
||||
drained := false
|
||||
preStopWait(func() { drained = true }, 0, nil)()
|
||||
if drained {
|
||||
t.Error("probes failed with no pre-stop delay")
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3PreStopWaitElapses(t *testing.T) {
|
||||
const delay = 50 * time.Millisecond
|
||||
drained := false
|
||||
started := time.Now()
|
||||
preStopWait(func() { drained = true }, delay, nil)()
|
||||
if !drained {
|
||||
t.Error("probes did not fail during the pre-stop delay")
|
||||
}
|
||||
if elapsed := time.Since(started); elapsed < delay {
|
||||
t.Fatalf("wait returned after %s, want at least %s", elapsed, delay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3PreStopDelay(t *testing.T) {
|
||||
seconds := func(n int) *int { return &n }
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
seconds *int
|
||||
want time.Duration
|
||||
}{
|
||||
{"unset", nil, 0},
|
||||
{"zero", seconds(0), 0},
|
||||
{"negative", seconds(-1), 0},
|
||||
{"set", seconds(20), 20 * time.Second},
|
||||
} {
|
||||
if got := (&S3Options{preStopSeconds: tc.seconds}).preStopDelay(); got != tc.want {
|
||||
t.Errorf("%s: preStopDelay() = %s, want %s", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -212,6 +212,7 @@ func init() {
|
||||
s3Options.readerCacheSizeMB = cmdServer.Flag.Int64("s3.readerCacheSizeMB", 0, "memory budget in MiB for downloaded and in-flight reader buffers across all S3 GETs (0 means unlimited)")
|
||||
s3Options.allowUntrustedRemoteEndpoints = cmdServer.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
|
||||
s3Options.iamSelfServiceRequiresGrant = cmdServer.Flag.Bool("s3.iam.selfServiceRequiresGrant", false, iamSelfServiceRequiresGrantUsage)
|
||||
s3Options.preStopSeconds = cmdServer.Flag.Int("s3.preStopSeconds", 0, s3PreStopSecondsUsage)
|
||||
|
||||
sftpOptions.port = cmdServer.Flag.Int("sftp.port", 2022, "SFTP server listen port")
|
||||
sftpOptions.idleTimeout = cmdServer.Flag.Int("sftp.idleTimeout", 10, "connection idle seconds")
|
||||
|
||||
@@ -115,6 +115,7 @@ type S3ApiServer struct {
|
||||
inFlightDataSize int64
|
||||
inFlightUploads int64
|
||||
inFlightDataLimitCond *sync.Cond
|
||||
draining atomic.Bool
|
||||
embeddedIam *EmbeddedIamApi // Embedded IAM API server (when enabled)
|
||||
stsHandlers *STSHandlers // STS HTTP handlers for AssumeRoleWithWebIdentity
|
||||
// icebergCredentialRole is the role the Iceberg catalog assumes to vend
|
||||
|
||||
@@ -13,6 +13,16 @@ func (s3a *S3ApiServer) StatusHandler(w http.ResponseWriter, r *http.Request) {
|
||||
s3err.WriteResponse(w, r, http.StatusOK, body, "application/json")
|
||||
return
|
||||
}
|
||||
if s3a.draining.Load() {
|
||||
s3err.WriteResponse(w, r, http.StatusServiceUnavailable, []byte{}, "")
|
||||
return
|
||||
}
|
||||
// write out the response code and content type header
|
||||
s3err.WriteResponse(w, r, http.StatusOK, []byte{}, "")
|
||||
}
|
||||
|
||||
// StartDraining makes the health probes fail so load balancers stop routing
|
||||
// new requests here, while requests that still arrive are served as usual.
|
||||
func (s3a *S3ApiServer) StartDraining() {
|
||||
s3a.draining.Store(true)
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStatusHandlerFailsWhileDraining(t *testing.T) {
|
||||
s3a := &S3ApiServer{}
|
||||
probe := func() int {
|
||||
recorder := httptest.NewRecorder()
|
||||
s3a.StatusHandler(recorder, httptest.NewRequest(http.MethodGet, "/healthz", nil))
|
||||
return recorder.Code
|
||||
}
|
||||
if code := probe(); code != http.StatusOK {
|
||||
t.Fatalf("probe returned %d before draining", code)
|
||||
}
|
||||
s3a.StartDraining()
|
||||
if code := probe(); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("probe returned %d while draining, want 503", code)
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
var signalChan chan os.Signal
|
||||
var interruptHooks = make([]func(), 0)
|
||||
var firstInterruptHooks = make([]func(), 0)
|
||||
var interruptHookLock sync.RWMutex
|
||||
var reloadHooks = make([]func(), 0)
|
||||
var reloadHookLock sync.RWMutex
|
||||
@@ -43,21 +44,29 @@ func init() {
|
||||
}
|
||||
reloadHookLock.RUnlock()
|
||||
} else {
|
||||
interruptHookLock.RLock()
|
||||
// Execute hooks in reverse registration order (LIFO/defer-style) so
|
||||
// later-started services shut down before the services they depend on.
|
||||
for i := len(interruptHooks) - 1; i >= 0; i-- {
|
||||
hook := interruptHooks[i]
|
||||
glog.V(4).Infof("exec interrupt hook func name:%s", GetFunctionName(hook))
|
||||
hook()
|
||||
}
|
||||
interruptHookLock.RUnlock()
|
||||
runInterruptHooks()
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// runInterruptHooks runs the OnInterruptFirst hooks in registration order,
|
||||
// then the OnInterrupt hooks in reverse registration order (LIFO/defer-style)
|
||||
// so later-started services shut down before the services they depend on.
|
||||
func runInterruptHooks() {
|
||||
interruptHookLock.RLock()
|
||||
defer interruptHookLock.RUnlock()
|
||||
for _, hook := range firstInterruptHooks {
|
||||
hook()
|
||||
}
|
||||
for i := len(interruptHooks) - 1; i >= 0; i-- {
|
||||
hook := interruptHooks[i]
|
||||
glog.V(4).Infof("exec interrupt hook func name:%s", GetFunctionName(hook))
|
||||
hook()
|
||||
}
|
||||
}
|
||||
|
||||
func OnReload(fn func()) {
|
||||
// prevent reentry
|
||||
reloadHookLock.Lock()
|
||||
@@ -74,3 +83,11 @@ func OnInterrupt(fn func()) {
|
||||
// controlling terminal close, daemon not exit
|
||||
interruptHooks = append(interruptHooks, fn)
|
||||
}
|
||||
|
||||
// OnInterruptFirst registers fn to run before every OnInterrupt hook,
|
||||
// whenever those were registered.
|
||||
func OnInterruptFirst(fn func()) {
|
||||
interruptHookLock.Lock()
|
||||
defer interruptHookLock.Unlock()
|
||||
firstInterruptHooks = append(firstInterruptHooks, fn)
|
||||
}
|
||||
@@ -4,3 +4,6 @@ package grace
|
||||
|
||||
func OnInterrupt(fn func()) {
|
||||
}
|
||||
|
||||
func OnInterruptFirst(fn func()) {
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
//go:build !plan9
|
||||
|
||||
package grace
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOnInterruptFirstRunsBeforeEarlierHooks(t *testing.T) {
|
||||
interruptHookLock.Lock()
|
||||
savedHooks, savedFirst := interruptHooks, firstInterruptHooks
|
||||
interruptHooks, firstInterruptHooks = nil, nil
|
||||
interruptHookLock.Unlock()
|
||||
t.Cleanup(func() {
|
||||
interruptHookLock.Lock()
|
||||
interruptHooks, firstInterruptHooks = savedHooks, savedFirst
|
||||
interruptHookLock.Unlock()
|
||||
})
|
||||
|
||||
var order []string
|
||||
OnInterrupt(func() { order = append(order, "volume") })
|
||||
OnInterruptFirst(func() { order = append(order, "s3 wait") })
|
||||
OnInterrupt(func() { order = append(order, "s3") })
|
||||
OnInterruptFirst(func() { order = append(order, "second wait") })
|
||||
|
||||
runInterruptHooks()
|
||||
|
||||
want := []string{"s3 wait", "second wait", "s3", "volume"}
|
||||
if !reflect.DeepEqual(order, want) {
|
||||
t.Fatalf("hooks ran as %v, want %v", order, want)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user