Fix/s3 abandoned chunk download (#12616)

* filer: cancel a chunk download that no reader waits for

* filer: stop waiting for reader cache budget once cancelled

* filer: keep a cancelled download's re-lookups off its context

The refresh and retry lookups took the download's context. FilerClient
coalesces concurrent lookups of a volume, so a cancelled download leading
one failed the other readers waiting on it. Fetches still stop on cancel.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: test that cancelling a download leaves its re-lookup running

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: cancel unheld downloads when the reader cache is destroyed

ReaderCache.destroy waited for every downloader, so closing a reader
waited for a stalled prefetch nobody read. Downloads with no reader or
stream on them are now cancelled first; ones still in use keep running.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: test that destroying the reader cache cancels an unread prefetch

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: cancel a destroyed download once its last holder leaves

destroy skipped downloads a reader or stream still held, and once they
were out of the map nothing cancelled them, so destroy waited for a fetch
nobody used. removeConsumedLocked now cancels any consumable download,
whether or not it is still in the map.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: test a destroyed download cancelled after its last stream leaves

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: release the stream only after destroy dropped the download

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* filer: test a cancelled stream leaving a waiting reader's download running

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-11-13 12:45:34 -08:00
1 parent 813ac74d79
commit d695d101b5
4 files changed
+515 -18

No files matched your search

+45 -17
View File
@@ -49,6 +49,9 @@ type SingleChunkCacher struct {
wg sync.WaitGroup
cacheStartedCh chan struct{}
done chan struct{} // signals when download is complete
// ctx is cancelled once no reader can use the download, or when it ends
ctx context.Context
cancel context.CancelFunc
}
func NewReaderCache(limit int, chunkCache chunk_cache.ChunkCache, lookupFileIdFn wdclient.LookupFileIdFunctionType, cacheInvalidator CacheInvalidator, budgets ...*ReaderCacheBudget) *ReaderCache {
@@ -392,12 +395,18 @@ func (rc *ReaderCache) removeConsumed(downloader *SingleChunkCacher) {
}
}
// removeConsumedLocked also cancels a running download, as no reader can attach to it any more,
// including one destroy already dropped from the map.
func (rc *ReaderCache) removeConsumedLocked(downloader *SingleChunkCacher) (removed bool) {
if rc.downloaders[downloader.chunkFileId] == downloader && downloader.consumable() {
delete(rc.downloaders, downloader.chunkFileId)
return true
if !downloader.consumable() {
return false
}
return false
downloader.cancel()
if rc.downloaders[downloader.chunkFileId] != downloader {
return false
}
delete(rc.downloaders, downloader.chunkFileId)
return true
}
func (rc *ReaderCache) destroy() {
@@ -405,13 +414,24 @@ func (rc *ReaderCache) destroy() {
downloaders := rc.downloaders
rc.downloaders = make(map[string]*SingleChunkCacher)
rc.Unlock()
// out of the map no stream can come back to a download, so it is cancelled
// now or once its last reader or stream leaves
for _, downloader := range downloaders {
atomic.StoreInt32(&downloader.left, 1)
if downloader.consumable() {
downloader.cancel()
}
}
for _, downloader := range downloaders {
downloader.destroy()
}
}
func newSingleChunkCacher(parent *ReaderCache, fileId string, cipherKey []byte, isGzipped bool, chunkSize int, shouldCache bool) *SingleChunkCacher {
ctx, cancel := context.WithCancel(context.Background())
return &SingleChunkCacher{
ctx: ctx,
cancel: cancel,
parent: parent,
chunkFileId: fileId,
cipherKey: cipherKey,
@@ -428,6 +448,7 @@ func newSingleChunkCacher(parent *ReaderCache, fileId string, cipherKey []byte,
// downloaders map, so a removal can never observe the fetch as absent.
func (s *SingleChunkCacher) startCaching() {
defer func() {
s.cancel()
close(s.done)
s.wg.Done()
if s.hasCompletedError() {
@@ -443,13 +464,7 @@ func (s *SingleChunkCacher) startCaching() {
return
}
// Intentionally use context.Background(), not a request-specific context.
// The downloaded chunk is a shared resource: multiple concurrent readers may
// wait on this same download via s.done. A request-scoped context that got
// cancelled would abort the download and error every other waiting reader.
// The download always runs to completion once started; readers that cancel
// individually drop out via readChunkAt's select on ctx.Done().
urlStrings, err := s.parent.lookupFileIdFn(context.Background(), s.chunkFileId)
urlStrings, err := s.lookupFileId(s.ctx, s.chunkFileId)
if err != nil {
s.setError(fmt.Errorf("operation LookupFileId %s failed, err: %v", s.chunkFileId, err))
return
@@ -459,9 +474,16 @@ func (s *SingleChunkCacher) startCaching() {
return
}
data, fetchErr := s.fetchChunkData(context.Background(), urlStrings)
// not a request's context: other readers may wait on this download via s.done
// and it is cancelled only when removeConsumedLocked drops the cacher
data, fetchErr := s.fetchChunkData(s.ctx, urlStrings)
if fetchErr != nil {
data, fetchErr = s.retryFetchAfterCacheInvalidation(context.Background(), urlStrings, fetchErr)
data, fetchErr = s.retryFetchAfterCacheInvalidation(s.ctx, urlStrings, fetchErr)
}
// a cancelled fetch can return a partly filled buffer without an error
if fetchErr == nil && s.ctx.Err() != nil {
mem.Free(data)
data, fetchErr = nil, s.ctx.Err()
}
// Now acquire lock to update state
@@ -498,7 +520,7 @@ func (s *SingleChunkCacher) fetchChunkData(ctx context.Context, urlStrings []str
// Allocate buffer and download without holding the lock.
// This allows multiple downloads to proceed in parallel.
data := mem.Allocate(s.chunkSize)
_, fetchErr := s.parent.fetchChunkDataFn(ctx, data, urlStrings, s.cipherKey, s.isGzipped, true, 0, s.chunkFileId, refreshUrls(ctx, s.parent.cacheInvalidator, s.parent.lookupFileIdFn, s.chunkFileId))
_, fetchErr := s.parent.fetchChunkDataFn(ctx, data, urlStrings, s.cipherKey, s.isGzipped, true, 0, s.chunkFileId, refreshUrls(ctx, s.parent.cacheInvalidator, s.lookupFileId, s.chunkFileId))
if fetchErr != nil {
mem.Free(data)
return nil, fetchErr
@@ -508,7 +530,7 @@ func (s *SingleChunkCacher) fetchChunkData(ctx context.Context, urlStrings []str
func (s *SingleChunkCacher) retryFetchAfterCacheInvalidation(ctx context.Context, oldUrlStrings []string, originalErr error) ([]byte, error) {
var data []byte
err := retryFetchWithFreshLocations(ctx, s.parent.cacheInvalidator, s.parent.lookupFileIdFn, s.chunkFileId, oldUrlStrings, originalErr, func(newUrls []string) error {
err := retryFetchWithFreshLocations(ctx, s.parent.cacheInvalidator, s.lookupFileId, s.chunkFileId, oldUrlStrings, originalErr, func(newUrls []string) error {
var fetchErr error
data, fetchErr = s.fetchChunkData(ctx, newUrls)
return fetchErr
@@ -519,6 +541,12 @@ func (s *SingleChunkCacher) retryFetchAfterCacheInvalidation(ctx context.Context
return data, nil
}
// lookupFileId ignores the download's context: FilerClient coalesces concurrent
// lookups of a volume, so a cancelled download must not fail other readers' lookups.
func (s *SingleChunkCacher) lookupFileId(_ context.Context, fileId string) ([]string, error) {
return s.parent.lookupFileIdFn(context.Background(), fileId)
}
func (s *SingleChunkCacher) destroy() {
// wait for all reads to finish before destroying the data
s.wg.Wait()
@@ -533,8 +561,8 @@ func (s *SingleChunkCacher) destroy() {
// readChunkAt reads data from the cached chunk.
// It waits for the download to complete if it's still in progress.
// The ctx parameter allows the reader to cancel its wait (but the download continues
// for other readers - see comment in startCaching about shared resource semantics).
// The ctx parameter allows the reader to cancel its wait; the download continues
// for the other readers and streams that still wait on it.
// The caller must s.wg.Add(1) under the ReaderCache lock before calling; this only releases it.
func (s *SingleChunkCacher) readChunkAt(ctx context.Context, buf []byte, offset int64) (n int, err error) {
defer func() {