Files
seaweedfs/weed/mount/dirty_pages_chunked.go
T
4f70ad015e mount: fail the write when the swap file chunk cannot be created (#12655)
* mount: fail the write when the swap file chunk cannot be created

#7957 added a nil check after NewSwapFileChunk so a failed swap file
creation returns an error instead of panicking. The result was assigned
to pageChunk, a PageChunk interface, before the check, and an interface
holding a nil *SwapFileChunk is not nil, so the check never fired:
SaveDataAt stored the nil chunk and panicked in WriteDataAt.

Check the *SwapFileChunk before converting it. The new test points the
swap directory below a regular file; on master it panics in
SwapFileChunk.WriteDataAt.

Signed-off-by: anant <170962632+antcybersec@users.noreply.github.com>

* mount: grow the file only after its write is buffered

Write set the new size, times, pending append offset and uncommitted
bytes before AddPage. When AddPage failed (now that a swap file chunk
that cannot be created returns an error), the write was reported as
failed but the larger size stayed on the handle's entry, so the next
flush published it with a hole where the data should be.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: report a partly buffered write as a short write

PageWriter.AddPage fills one chunk at a time, so when a later chunk
cannot be created the earlier chunks already hold their bytes. Return
how much was buffered, and let Write grow the size and times over that
prefix and report it, instead of failing the whole write while the
prefix still reaches the next flush.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: report the bytes a failed copy_file_range already buffered

A streaming copy whose page fails partway returned an error even though
earlier pages, or the start of the failing one, were already buffered and
would reach the next flush with the destination size left short. Count
the buffered bytes, grow the destination over them, and return them as a
short copy.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: free the copy test's destination pages

The buffered memory chunk counts toward the process-wide memory chunk
ceiling, so repeated runs of the test pushed later ones onto the
unusable swap directory.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: count only the bytes a swap chunk stored

A swap chunk whose file write fails stores a prefix or nothing, but
SaveDataAt returned no error and AddPage counted the whole span as
buffered. SaveDataAt now reports the short write, and AddPage passes the
stored count up so a write is never acknowledged past it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: fail a partly buffered writeback write

Kernel writeback ignores the count in a WRITE reply, so a short write
there drops the unbuffered rest without an error. A write flagged as
page-cache writeback now keeps its buffered prefix but returns the
error, which the kernel reports on fsync.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Signed-off-by: anant <170962632+antcybersec@users.noreply.github.com>
Co-authored-by: anant <170962632+antcybersec@users.noreply.github.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 19:34:43 -08:00

186 lines
5.9 KiB
Go

package mount
import (
"errors"
"fmt"
"io"
"sync"
"sync/atomic"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/mount/page_writer"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
type ChunkedDirtyPages struct {
fh *FileHandle
writeWaitGroup sync.WaitGroup
lastErrLock sync.Mutex
lastErr error
collection string
replication string
uploadPipeline *page_writer.UploadPipeline
hasWrites bool
saveDataAsChunk func(util.FullPath) filer.SaveDataAsChunkFunctionType
destroyed atomic.Bool
// attachLock keeps Destroy from landing between the destroyed check and
// the chunk append, which would put old bytes into the replacing entry.
attachLock sync.Mutex
}
var (
_ = page_writer.DirtyPages(&ChunkedDirtyPages{})
)
var errDirtyPagesDiscarded = errors.New("dirty pages discarded")
func newMemoryChunkPages(fh *FileHandle, chunkSize int64) *ChunkedDirtyPages {
dirtyPages := &ChunkedDirtyPages{
fh: fh,
saveDataAsChunk: fh.wfs.saveDataAsChunk,
}
swapFileDir := fh.wfs.option.getUniqueCacheDirForWrite()
dirtyPages.uploadPipeline = page_writer.NewUploadPipeline(fh.wfs.concurrentWriters, chunkSize,
dirtyPages.saveChunkedFileIntervalToStorage, fh.wfs.option.ConcurrentWriters, swapFileDir,
fh.wfs.writeBufferAccountant)
return dirtyPages
}
func (pages *ChunkedDirtyPages) AddPage(offset int64, data []byte, isSequential bool, tsNs int64) (n int, err error) {
// A failed chunk upload already dropped its data, so this handle can never
// flush cleanly. Reject further writes so the writer fails fast instead of
// pushing the rest of the file through a pipeline that cannot persist it.
if err := pages.LastError(); err != nil {
return 0, err
}
pages.hasWrites = true
glog.V(4).Infof("%v memory AddPage [%d, %d)", pages.fh.fh, offset, offset+int64(len(data)))
return pages.uploadPipeline.SaveDataAt(data, offset, isSequential, tsNs)
}
func (pages *ChunkedDirtyPages) HasWrites() bool {
return pages.hasWrites
}
func (pages *ChunkedDirtyPages) FlushData() error {
if !pages.hasWrites {
return nil
}
pages.uploadPipeline.FlushAll()
if err := pages.LastError(); err != nil {
return fmt.Errorf("flush data: %w", err)
}
return nil
}
// LastError returns the first chunk upload failure on this handle. It is
// sticky: the failed chunk's data is gone, so the handle stays poisoned.
func (pages *ChunkedDirtyPages) LastError() error {
pages.lastErrLock.Lock()
defer pages.lastErrLock.Unlock()
return pages.lastErr
}
func (pages *ChunkedDirtyPages) setLastError(err error) {
pages.lastErrLock.Lock()
defer pages.lastErrLock.Unlock()
if pages.lastErr == nil {
pages.lastErr = err
}
}
func (pages *ChunkedDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64, tsNs int64) (maxStop int64) {
if !pages.hasWrites {
return
}
return pages.uploadPipeline.MaybeReadDataAt(data, startOffset, tsNs)
}
func (pages *ChunkedDirtyPages) saveChunkedFileIntervalToStorage(reader io.Reader, offset int64, size int64, modifiedTsNs int64, cleanupFn func()) {
defer cleanupFn()
fileFullPath := pages.fh.FullPath()
fileName := fileFullPath.Name()
save := pages.saveDataAsChunk(fileFullPath)
var chunk *filer_pb.FileChunk
err := util.RetryUntil("writeback upload "+string(fileFullPath), func() (err error) {
if pages.destroyed.Load() {
return errDirtyPagesDiscarded
}
chunk, err = save(reader, fileName, offset, modifiedTsNs, uint64(size))
return err
}, func(err error) bool {
// close() already returned for a writeback handle, so nothing would
// see the error and the buffered data is the only copy left.
if !pages.fh.wfs.option.WritebackCache || !isRetryableUploadError(err) || pages.destroyed.Load() {
return false
}
seeker, ok := reader.(io.Seeker)
if !ok {
return false
}
if _, seekErr := seeker.Seek(0, io.SeekStart); seekErr != nil {
return false
}
glog.Warningf("%v writeback upload [%d,%d) failed, keeping the data and retrying: %v", fileFullPath, offset, offset+size, err)
return true
})
if err != nil {
glog.V(0).Infof("%v saveToStorage [%d,%d): %v", fileFullPath, offset, offset+size, err)
pages.setLastError(err)
return
}
// Truncation or newer content replaced these pages while they uploaded.
pages.attachLock.Lock()
defer pages.attachLock.Unlock()
if pages.destroyed.Load() {
glog.V(3).Infof("%v saveToStorage %s [%d,%d): pages discarded, chunk not attached", fileFullPath, chunk.FileId, offset, offset+size)
return
}
pages.fh.AddChunks([]*filer_pb.FileChunk{chunk})
pages.fh.entryChunkGroup.AddChunk(chunk)
glog.V(3).Infof("%v saveToStorage %s [%d,%d)", fileFullPath, chunk.FileId, offset, offset+size)
}
// isRetryableUploadError also takes the failures the upload layer reassigns on,
// such as a filer proxy answering 500, but never a read-only verdict.
func isRetryableUploadError(err error) bool {
if errors.Is(err, filer_pb.ErrReadOnly) {
return false
}
return util.IsTransientError(err) || operation.ShouldReassignUpload(err)
}
func (pages *ChunkedDirtyPages) Destroy() {
pages.attachLock.Lock()
pages.destroyed.Store(true)
pages.attachLock.Unlock()
pages.uploadPipeline.Shutdown()
}
func (pages *ChunkedDirtyPages) EvictOneWritableChunk() bool {
return pages.uploadPipeline.EvictOneWritableChunk()
}
func (pages *ChunkedDirtyPages) ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio int64, frontierLag int, isSequential bool) bool {
return pages.uploadPipeline.ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio, frontierLag, isSequential)
}
func (pages *ChunkedDirtyPages) LockForRead(startOffset, stopOffset int64) {
pages.uploadPipeline.LockForRead(startOffset, stopOffset)
}
func (pages *ChunkedDirtyPages) UnlockForRead(startOffset, stopOffset int64) {
pages.uploadPipeline.UnlockForRead(startOffset, stopOffset)
}