Files
seaweedfs/weed/mount/dirty_pages_chunked_test.go
T
4f70ad015e mount: fail the write when the swap file chunk cannot be created (#12655)
* mount: fail the write when the swap file chunk cannot be created

#7957 added a nil check after NewSwapFileChunk so a failed swap file
creation returns an error instead of panicking. The result was assigned
to pageChunk, a PageChunk interface, before the check, and an interface
holding a nil *SwapFileChunk is not nil, so the check never fired:
SaveDataAt stored the nil chunk and panicked in WriteDataAt.

Check the *SwapFileChunk before converting it. The new test points the
swap directory below a regular file; on master it panics in
SwapFileChunk.WriteDataAt.

Signed-off-by: anant <170962632+antcybersec@users.noreply.github.com>

* mount: grow the file only after its write is buffered

Write set the new size, times, pending append offset and uncommitted
bytes before AddPage. When AddPage failed (now that a swap file chunk
that cannot be created returns an error), the write was reported as
failed but the larger size stayed on the handle's entry, so the next
flush published it with a hole where the data should be.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: report a partly buffered write as a short write

PageWriter.AddPage fills one chunk at a time, so when a later chunk
cannot be created the earlier chunks already hold their bytes. Return
how much was buffered, and let Write grow the size and times over that
prefix and report it, instead of failing the whole write while the
prefix still reaches the next flush.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: report the bytes a failed copy_file_range already buffered

A streaming copy whose page fails partway returned an error even though
earlier pages, or the start of the failing one, were already buffered and
would reach the next flush with the destination size left short. Count
the buffered bytes, grow the destination over them, and return them as a
short copy.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: free the copy test's destination pages

The buffered memory chunk counts toward the process-wide memory chunk
ceiling, so repeated runs of the test pushed later ones onto the
unusable swap directory.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: count only the bytes a swap chunk stored

A swap chunk whose file write fails stores a prefix or nothing, but
SaveDataAt returned no error and AddPage counted the whole span as
buffered. SaveDataAt now reports the short write, and AddPage passes the
stored count up so a write is never acknowledged past it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* mount: fail a partly buffered writeback write

Kernel writeback ignores the count in a WRITE reply, so a short write
there drops the unbuffered rest without an error. A write flagged as
page-cache writeback now keeps its buffered prefix but returns the
error, which the kernel reports on fsync.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Signed-off-by: anant <170962632+antcybersec@users.noreply.github.com>
Co-authored-by: anant <170962632+antcybersec@users.noreply.github.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 19:34:43 -08:00

190 lines
6.8 KiB
Go

package mount
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"syscall"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// Once a chunk upload fails its data is gone, so the handle must reject
// further writes instead of buffering the rest of the file — against a full
// cluster that turned cp into an hours-long crawl that only errored at close.
func TestChunkedDirtyPagesFailWritesAfterUploadError(t *testing.T) {
fh := &FileHandle{wfs: &WFS{option: &Option{}}}
pages := newMemoryChunkPages(fh, 1024)
defer pages.Destroy()
pages.hasWrites = true
uploadErr := errors.New("assign volume failure: no writable volumes")
pages.setLastError(uploadErr)
if _, err := pages.AddPage(0, []byte("x"), true, 1); !errors.Is(err, uploadErr) {
t.Fatalf("AddPage after upload failure = %v, want sticky %v", err, uploadErr)
}
if err := pages.FlushData(); !errors.Is(err, uploadErr) {
t.Fatalf("FlushData after upload failure = %v, want wrapped %v", err, uploadErr)
}
// First failure wins; later errors must not mask the root cause.
pages.setLastError(errors.New("later error"))
if err := pages.LastError(); !errors.Is(err, uploadErr) {
t.Fatalf("LastError = %v, want first error %v", err, uploadErr)
}
}
type flakySave struct {
failures []error
calls int
bodies []string
onCall func()
}
func (s *flakySave) saveDataAsChunk(util.FullPath) filer.SaveDataAsChunkFunctionType {
return func(reader io.Reader, name string, offset int64, tsNs int64, _ uint64) (*filer_pb.FileChunk, error) {
body, _ := io.ReadAll(reader)
s.bodies = append(s.bodies, string(body))
s.calls++
if s.onCall != nil {
s.onCall()
}
if s.calls <= len(s.failures) {
return nil, s.failures[s.calls-1]
}
return &filer_pb.FileChunk{FileId: "1,01", Offset: offset, Size: uint64(len(body)), ModifiedTsNs: tsNs}, nil
}
}
func newUploadTestPages(t *testing.T, writeback bool, save *flakySave) *ChunkedDirtyPages {
t.Helper()
wfs := newInvalidateTestWFS(t)
wfs.option.WritebackCache = writeback
fh := newFileHandle(wfs, 1, 2, &filer_pb.Entry{Name: "f", Attributes: &filer_pb.FuseAttributes{}})
fh.RememberPath("/dir/f")
pages := newMemoryChunkPages(fh, 1024)
t.Cleanup(pages.Destroy)
pages.saveDataAsChunk = save.saveDataAsChunk
return pages
}
var errVolumeRefused = fmt.Errorf("upload data: %w", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED})
// filerProxyOutage is what a filer proxy sends when it cannot reach the
// volume server: a 500 with an empty body.
func filerProxyOutage(t *testing.T) error {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer server.Close()
_, err := operation.NewUploaderWithHttpClient(server.Client()).UploadData(context.Background(), []byte("hello"),
&operation.UploadOption{UploadUrl: server.URL, MaxAttempts: 1})
if err == nil {
t.Fatal("upload to the failing proxy succeeded")
}
return fmt.Errorf("upload data: %w", err)
}
// close() has already returned for a writeback handle, so a volume outage must
// not drop the buffered chunk: the same bytes go out again once it ends.
func TestChunkedDirtyPagesWritebackRetriesTransientUploadError(t *testing.T) {
for name, outage := range map[string]error{
"volume refused": errVolumeRefused,
"filer proxy 500": filerProxyOutage(t),
// With no volume server left the assign keeps timing out, and the
// filer client gives up after its own retries.
"assign timed out": fmt.Errorf("upload data: filerGrpcAddress assign volume: %w",
status.Error(codes.Unavailable, "filer 127.0.0.1:18888: call timed out")),
} {
t.Run(name, func(t *testing.T) {
save := &flakySave{failures: []error{outage}}
pages := newUploadTestPages(t, true, save)
pages.saveChunkedFileIntervalToStorage(util.NewBytesReader([]byte("hello")), 0, 5, 1, func() {})
if err := pages.LastError(); err != nil {
t.Fatalf("LastError = %v, want the retried upload to succeed", err)
}
if save.calls != 2 || save.bodies[1] != "hello" {
t.Fatalf("uploads = %d with bodies %q, want a second upload of the same bytes", save.calls, save.bodies)
}
if chunks := pages.fh.GetEntry().GetChunks(); len(chunks) != 1 || chunks[0].Size != 5 {
t.Fatalf("entry chunks = %v, want the retried chunk", chunks)
}
})
}
}
func TestChunkedDirtyPagesUploadErrorWithoutRetry(t *testing.T) {
permanent := errors.New("upload data: assign volume failure: no writable volumes")
readOnly := fmt.Errorf("upload data: assign volume failure path:\"/unavailable/f\": %w",
filer_pb.AssignVolumeResponseError(&filer_pb.AssignVolumeResponse{
Error: "assign volume: read only: /unavailable",
ErrorCode: filer_pb.FilerError_READ_ONLY,
}))
tests := []struct {
name string
writeback bool
err error
discard func(pages *ChunkedDirtyPages)
wantErr error
}{
{name: "sync mode reports the outage to close", err: errVolumeRefused},
{name: "writeback permanent error", writeback: true, err: permanent},
{name: "writeback read-only path", writeback: true, err: readOnly},
{name: "writeback pages discarded mid-upload", writeback: true, err: errVolumeRefused,
discard: func(pages *ChunkedDirtyPages) { pages.Destroy() }},
{name: "writeback pages discarded during the retry wait", writeback: true, err: errVolumeRefused,
discard: func(pages *ChunkedDirtyPages) { time.AfterFunc(100*time.Millisecond, pages.Destroy) },
wantErr: errDirtyPagesDiscarded},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
save := &flakySave{failures: []error{tt.err}}
pages := newUploadTestPages(t, tt.writeback, save)
if tt.discard != nil {
save.onCall = func() { tt.discard(pages) }
}
wantErr := tt.err
if tt.wantErr != nil {
wantErr = tt.wantErr
}
pages.saveChunkedFileIntervalToStorage(util.NewBytesReader([]byte("hello")), 0, 5, 1, func() {})
if save.calls != 1 {
t.Fatalf("uploads = %d, want 1", save.calls)
}
if err := pages.LastError(); !errors.Is(err, wantErr) {
t.Fatalf("LastError = %v, want %v", err, wantErr)
}
})
}
}
// A chunk whose pages were discarded while it uploaded must not reappear in
// the handle's entry, e.g. after a truncate replaced the content.
func TestChunkedDirtyPagesDiscardedUploadNotAttached(t *testing.T) {
save := &flakySave{}
pages := newUploadTestPages(t, true, save)
save.onCall = pages.Destroy
pages.saveChunkedFileIntervalToStorage(util.NewBytesReader([]byte("hello")), 0, 5, 1, func() {})
if chunks := pages.fh.GetEntry().GetChunks(); len(chunks) != 0 {
t.Fatalf("entry chunks = %v, want none from discarded pages", chunks)
}
}