mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-17 01:05:29 +01:00
Fix/s3 abandoned chunk download (#12616)
* filer: cancel a chunk download that no reader waits for * filer: stop waiting for reader cache budget once cancelled * filer: keep a cancelled download's re-lookups off its context The refresh and retry lookups took the download's context. FilerClient coalesces concurrent lookups of a volume, so a cancelled download leading one failed the other readers waiting on it. Fetches still stop on cancel. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test that cancelling a download leaves its re-lookup running Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: cancel unheld downloads when the reader cache is destroyed ReaderCache.destroy waited for every downloader, so closing a reader waited for a stalled prefetch nobody read. Downloads with no reader or stream on them are now cancelled first; ones still in use keep running. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test that destroying the reader cache cancels an unread prefetch Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: cancel a destroyed download once its last holder leaves destroy skipped downloads a reader or stream still held, and once they were out of the map nothing cancelled them, so destroy waited for a fetch nobody used. removeConsumedLocked now cancels any consumable download, whether or not it is still in the map. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test a destroyed download cancelled after its last stream leaves Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: release the stream only after destroy dropped the download Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test a cancelled stream leaving a waiting reader's download running Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
4 files changed
+515
-18
No files matched your search
+45
-17
@@ -49,6 +49,9 @@ type SingleChunkCacher struct {
|
||||
wg sync.WaitGroup
|
||||
cacheStartedCh chan struct{}
|
||||
done chan struct{} // signals when download is complete
|
||||
// ctx is cancelled once no reader can use the download, or when it ends
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func NewReaderCache(limit int, chunkCache chunk_cache.ChunkCache, lookupFileIdFn wdclient.LookupFileIdFunctionType, cacheInvalidator CacheInvalidator, budgets ...*ReaderCacheBudget) *ReaderCache {
|
||||
@@ -392,12 +395,18 @@ func (rc *ReaderCache) removeConsumed(downloader *SingleChunkCacher) {
|
||||
}
|
||||
}
|
||||
|
||||
// removeConsumedLocked also cancels a running download, as no reader can attach to it any more,
|
||||
// including one destroy already dropped from the map.
|
||||
func (rc *ReaderCache) removeConsumedLocked(downloader *SingleChunkCacher) (removed bool) {
|
||||
if rc.downloaders[downloader.chunkFileId] == downloader && downloader.consumable() {
|
||||
delete(rc.downloaders, downloader.chunkFileId)
|
||||
return true
|
||||
if !downloader.consumable() {
|
||||
return false
|
||||
}
|
||||
return false
|
||||
downloader.cancel()
|
||||
if rc.downloaders[downloader.chunkFileId] != downloader {
|
||||
return false
|
||||
}
|
||||
delete(rc.downloaders, downloader.chunkFileId)
|
||||
return true
|
||||
}
|
||||
|
||||
func (rc *ReaderCache) destroy() {
|
||||
@@ -405,13 +414,24 @@ func (rc *ReaderCache) destroy() {
|
||||
downloaders := rc.downloaders
|
||||
rc.downloaders = make(map[string]*SingleChunkCacher)
|
||||
rc.Unlock()
|
||||
// out of the map no stream can come back to a download, so it is cancelled
|
||||
// now or once its last reader or stream leaves
|
||||
for _, downloader := range downloaders {
|
||||
atomic.StoreInt32(&downloader.left, 1)
|
||||
if downloader.consumable() {
|
||||
downloader.cancel()
|
||||
}
|
||||
}
|
||||
for _, downloader := range downloaders {
|
||||
downloader.destroy()
|
||||
}
|
||||
}
|
||||
|
||||
func newSingleChunkCacher(parent *ReaderCache, fileId string, cipherKey []byte, isGzipped bool, chunkSize int, shouldCache bool) *SingleChunkCacher {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &SingleChunkCacher{
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
parent: parent,
|
||||
chunkFileId: fileId,
|
||||
cipherKey: cipherKey,
|
||||
@@ -428,6 +448,7 @@ func newSingleChunkCacher(parent *ReaderCache, fileId string, cipherKey []byte,
|
||||
// downloaders map, so a removal can never observe the fetch as absent.
|
||||
func (s *SingleChunkCacher) startCaching() {
|
||||
defer func() {
|
||||
s.cancel()
|
||||
close(s.done)
|
||||
s.wg.Done()
|
||||
if s.hasCompletedError() {
|
||||
@@ -443,13 +464,7 @@ func (s *SingleChunkCacher) startCaching() {
|
||||
return
|
||||
}
|
||||
|
||||
// Intentionally use context.Background(), not a request-specific context.
|
||||
// The downloaded chunk is a shared resource: multiple concurrent readers may
|
||||
// wait on this same download via s.done. A request-scoped context that got
|
||||
// cancelled would abort the download and error every other waiting reader.
|
||||
// The download always runs to completion once started; readers that cancel
|
||||
// individually drop out via readChunkAt's select on ctx.Done().
|
||||
urlStrings, err := s.parent.lookupFileIdFn(context.Background(), s.chunkFileId)
|
||||
urlStrings, err := s.lookupFileId(s.ctx, s.chunkFileId)
|
||||
if err != nil {
|
||||
s.setError(fmt.Errorf("operation LookupFileId %s failed, err: %v", s.chunkFileId, err))
|
||||
return
|
||||
@@ -459,9 +474,16 @@ func (s *SingleChunkCacher) startCaching() {
|
||||
return
|
||||
}
|
||||
|
||||
data, fetchErr := s.fetchChunkData(context.Background(), urlStrings)
|
||||
// not a request's context: other readers may wait on this download via s.done
|
||||
// and it is cancelled only when removeConsumedLocked drops the cacher
|
||||
data, fetchErr := s.fetchChunkData(s.ctx, urlStrings)
|
||||
if fetchErr != nil {
|
||||
data, fetchErr = s.retryFetchAfterCacheInvalidation(context.Background(), urlStrings, fetchErr)
|
||||
data, fetchErr = s.retryFetchAfterCacheInvalidation(s.ctx, urlStrings, fetchErr)
|
||||
}
|
||||
// a cancelled fetch can return a partly filled buffer without an error
|
||||
if fetchErr == nil && s.ctx.Err() != nil {
|
||||
mem.Free(data)
|
||||
data, fetchErr = nil, s.ctx.Err()
|
||||
}
|
||||
|
||||
// Now acquire lock to update state
|
||||
@@ -498,7 +520,7 @@ func (s *SingleChunkCacher) fetchChunkData(ctx context.Context, urlStrings []str
|
||||
// Allocate buffer and download without holding the lock.
|
||||
// This allows multiple downloads to proceed in parallel.
|
||||
data := mem.Allocate(s.chunkSize)
|
||||
_, fetchErr := s.parent.fetchChunkDataFn(ctx, data, urlStrings, s.cipherKey, s.isGzipped, true, 0, s.chunkFileId, refreshUrls(ctx, s.parent.cacheInvalidator, s.parent.lookupFileIdFn, s.chunkFileId))
|
||||
_, fetchErr := s.parent.fetchChunkDataFn(ctx, data, urlStrings, s.cipherKey, s.isGzipped, true, 0, s.chunkFileId, refreshUrls(ctx, s.parent.cacheInvalidator, s.lookupFileId, s.chunkFileId))
|
||||
if fetchErr != nil {
|
||||
mem.Free(data)
|
||||
return nil, fetchErr
|
||||
@@ -508,7 +530,7 @@ func (s *SingleChunkCacher) fetchChunkData(ctx context.Context, urlStrings []str
|
||||
|
||||
func (s *SingleChunkCacher) retryFetchAfterCacheInvalidation(ctx context.Context, oldUrlStrings []string, originalErr error) ([]byte, error) {
|
||||
var data []byte
|
||||
err := retryFetchWithFreshLocations(ctx, s.parent.cacheInvalidator, s.parent.lookupFileIdFn, s.chunkFileId, oldUrlStrings, originalErr, func(newUrls []string) error {
|
||||
err := retryFetchWithFreshLocations(ctx, s.parent.cacheInvalidator, s.lookupFileId, s.chunkFileId, oldUrlStrings, originalErr, func(newUrls []string) error {
|
||||
var fetchErr error
|
||||
data, fetchErr = s.fetchChunkData(ctx, newUrls)
|
||||
return fetchErr
|
||||
@@ -519,6 +541,12 @@ func (s *SingleChunkCacher) retryFetchAfterCacheInvalidation(ctx context.Context
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// lookupFileId ignores the download's context: FilerClient coalesces concurrent
|
||||
// lookups of a volume, so a cancelled download must not fail other readers' lookups.
|
||||
func (s *SingleChunkCacher) lookupFileId(_ context.Context, fileId string) ([]string, error) {
|
||||
return s.parent.lookupFileIdFn(context.Background(), fileId)
|
||||
}
|
||||
|
||||
func (s *SingleChunkCacher) destroy() {
|
||||
// wait for all reads to finish before destroying the data
|
||||
s.wg.Wait()
|
||||
@@ -533,8 +561,8 @@ func (s *SingleChunkCacher) destroy() {
|
||||
|
||||
// readChunkAt reads data from the cached chunk.
|
||||
// It waits for the download to complete if it's still in progress.
|
||||
// The ctx parameter allows the reader to cancel its wait (but the download continues
|
||||
// for other readers - see comment in startCaching about shared resource semantics).
|
||||
// The ctx parameter allows the reader to cancel its wait; the download continues
|
||||
// for the other readers and streams that still wait on it.
|
||||
// The caller must s.wg.Add(1) under the ReaderCache lock before calling; this only releases it.
|
||||
func (s *SingleChunkCacher) readChunkAt(ctx context.Context, buf []byte, offset int64) (n int, err error) {
|
||||
defer func() {
|
||||
|
||||
Reference in new issue
Block a user