mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-07 14:57:48 +02:00
c739e5cf78
Standalone weed s3 only shut its servers down when shutdownCtx was set, which only weed mini does. On SIGTERM the interrupt hooks ran and the process exited with requests still in flight, so clients saw connection resets during rolling restarts. Register an interrupt hook that drains every S3 HTTP(S) listener (including the local and unix socket ones, and the Iceberg and Lance servers) for up to 15s alongside a bounded gRPC GracefulStop, then closes the S3 API server, reusing the filer's shutdown helper. Serve exits now join that shutdown so the process does not exit mid-drain, and the secondary listeners tolerate http.ErrServerClosed instead of exiting fatally. Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
82 lines
2.1 KiB
Go
82 lines
2.1 KiB
Go
package command
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api"
|
|
)
|
|
|
|
func TestS3ShutdownDrainsActiveRequestBeforeServeReturns(t *testing.T) {
|
|
entered := make(chan struct{})
|
|
release := make(chan struct{})
|
|
finish := sync.OnceFunc(func() { close(release) })
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
close(entered)
|
|
<-release
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}))
|
|
t.Cleanup(server.Close)
|
|
t.Cleanup(finish)
|
|
httpClosing := make(chan struct{})
|
|
server.Config.RegisterOnShutdown(func() { close(httpClosing) })
|
|
|
|
requestDone := make(chan struct{})
|
|
go func() {
|
|
defer close(requestDone)
|
|
response, err := server.Client().Get(server.URL)
|
|
if err != nil {
|
|
t.Error(err)
|
|
return
|
|
}
|
|
response.Body.Close()
|
|
if response.StatusCode != http.StatusNoContent {
|
|
t.Errorf("active request returned %s", response.Status)
|
|
}
|
|
}()
|
|
select {
|
|
case <-entered:
|
|
case <-requestDone:
|
|
t.Fatal("request ended before reaching the handler")
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("request did not reach the handler")
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
grpcStopped := make(chan struct{})
|
|
s3opt := &S3Options{shutdownCtx: ctx}
|
|
shutdown := s3opt.newShutdown(func() { close(grpcStopped) }, &s3api.S3ApiServer{}, []*http.Server{server.Config})
|
|
|
|
cancel()
|
|
select {
|
|
case <-httpClosing:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("cancelling shutdownCtx did not start the HTTP drain")
|
|
}
|
|
<-grpcStopped
|
|
|
|
// Model the main Serve path joining shutdown once its listener closes.
|
|
serveReturned := make(chan struct{})
|
|
go func() { shutdown(); close(serveReturned) }()
|
|
select {
|
|
case <-serveReturned:
|
|
t.Fatal("Serve exit path returned while a request was still active")
|
|
case <-requestDone:
|
|
t.Fatal("active request ended before it was released")
|
|
case <-time.After(100 * time.Millisecond):
|
|
}
|
|
|
|
finish()
|
|
<-requestDone
|
|
select {
|
|
case <-serveReturned:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("Serve exit path did not return after the request drained")
|
|
}
|
|
}
|