From 280f620b5592e6c5ed009d07c59b09106c391ccc Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 27 Apr 2026 01:11:04 -0700 Subject: [PATCH] fix(s3api): return 503 with Retry-After when remote object not cached yet When a GET hits a remote-only object whose cache fill timed out or was canceled, the handler returned 500 InternalError. SDK clients treat 500 as a server bug and surface it as a fatal error (boto3 S3DownloadFailedError), even though the cache is often still filling in the background and the next request would succeed. Return 503 ServiceUnavailable with Retry-After: 5 instead, matching AWS S3's "try again later" semantics. AWS SDKs already classify 503 as retryable and apply exponential backoff transparently, so clients recover without changes. Refs https://github.com/seaweedfs/seaweedfs/discussions/9174 --- weed/s3api/s3api_object_handlers.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 581bf761ba..012e982f17 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -1000,10 +1000,14 @@ func (s3a *S3ApiServer) streamFromVolumeServers(w http.ResponseWriter, r *http.R entry = cachedEntry glog.V(1).Infof("streamFromVolumeServers: successfully cached remote object, got %d chunks", len(chunks)) } else { - // Caching failed - return error to client - glog.Errorf("streamFromVolumeServers: failed to cache remote object for streaming") - s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) - return newStreamErrorWithResponse(fmt.Errorf("failed to cache remote object for streaming")) + // Cache attempt did not produce chunks in time. The cache may still + // be filling in the background, so return 503 with Retry-After to + // let S3 SDKs back off and retry transparently rather than treating + // this as a fatal 500 InternalError. + glog.V(1).Infof("streamFromVolumeServers: remote object %s/%s not cached yet, returning 503 for retry", bucket, object) + w.Header().Set("Retry-After", "5") + s3err.WriteErrorResponse(w, r, s3err.ErrServiceUnavailable) + return newStreamErrorWithResponse(fmt.Errorf("remote object not cached yet")) } } else if totalSize > 0 && len(entry.Content) == 0 { // Not a remote entry but has size without content - this is a data integrity issue