mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 08:45:22 +01:00
* mount: fail the write when the swap file chunk cannot be created #7957 added a nil check after NewSwapFileChunk so a failed swap file creation returns an error instead of panicking. The result was assigned to pageChunk, a PageChunk interface, before the check, and an interface holding a nil *SwapFileChunk is not nil, so the check never fired: SaveDataAt stored the nil chunk and panicked in WriteDataAt. Check the *SwapFileChunk before converting it. The new test points the swap directory below a regular file; on master it panics in SwapFileChunk.WriteDataAt. Signed-off-by: anant <170962632+antcybersec@users.noreply.github.com> * mount: grow the file only after its write is buffered Write set the new size, times, pending append offset and uncommitted bytes before AddPage. When AddPage failed (now that a swap file chunk that cannot be created returns an error), the write was reported as failed but the larger size stayed on the handle's entry, so the next flush published it with a hole where the data should be. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: report a partly buffered write as a short write PageWriter.AddPage fills one chunk at a time, so when a later chunk cannot be created the earlier chunks already hold their bytes. Return how much was buffered, and let Write grow the size and times over that prefix and report it, instead of failing the whole write while the prefix still reaches the next flush. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: report the bytes a failed copy_file_range already buffered A streaming copy whose page fails partway returned an error even though earlier pages, or the start of the failing one, were already buffered and would reach the next flush with the destination size left short. Count the buffered bytes, grow the destination over them, and return them as a short copy. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: free the copy test's destination pages The buffered memory chunk counts toward the process-wide memory chunk ceiling, so repeated runs of the test pushed later ones onto the unusable swap directory. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: count only the bytes a swap chunk stored A swap chunk whose file write fails stores a prefix or nothing, but SaveDataAt returned no error and AddPage counted the whole span as buffered. SaveDataAt now reports the short write, and AddPage passes the stored count up so a write is never acknowledged past it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: fail a partly buffered writeback write Kernel writeback ignores the count in a WRITE reply, so a short write there drops the unbuffered rest without an error. A write flagged as page-cache writeback now keeps its buffered prefix but returns the error, which the kernel reports on fsync. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Signed-off-by: anant <170962632+antcybersec@users.noreply.github.com> Co-authored-by: anant <170962632+antcybersec@users.noreply.github.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
190 lines
6.8 KiB
Go
190 lines
6.8 KiB
Go
package mount
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
|
"github.com/seaweedfs/seaweedfs/weed/operation"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
)
|
|
|
|
// Once a chunk upload fails its data is gone, so the handle must reject
|
|
// further writes instead of buffering the rest of the file — against a full
|
|
// cluster that turned cp into an hours-long crawl that only errored at close.
|
|
func TestChunkedDirtyPagesFailWritesAfterUploadError(t *testing.T) {
|
|
fh := &FileHandle{wfs: &WFS{option: &Option{}}}
|
|
pages := newMemoryChunkPages(fh, 1024)
|
|
defer pages.Destroy()
|
|
pages.hasWrites = true
|
|
|
|
uploadErr := errors.New("assign volume failure: no writable volumes")
|
|
pages.setLastError(uploadErr)
|
|
|
|
if _, err := pages.AddPage(0, []byte("x"), true, 1); !errors.Is(err, uploadErr) {
|
|
t.Fatalf("AddPage after upload failure = %v, want sticky %v", err, uploadErr)
|
|
}
|
|
if err := pages.FlushData(); !errors.Is(err, uploadErr) {
|
|
t.Fatalf("FlushData after upload failure = %v, want wrapped %v", err, uploadErr)
|
|
}
|
|
// First failure wins; later errors must not mask the root cause.
|
|
pages.setLastError(errors.New("later error"))
|
|
if err := pages.LastError(); !errors.Is(err, uploadErr) {
|
|
t.Fatalf("LastError = %v, want first error %v", err, uploadErr)
|
|
}
|
|
}
|
|
|
|
type flakySave struct {
|
|
failures []error
|
|
calls int
|
|
bodies []string
|
|
onCall func()
|
|
}
|
|
|
|
func (s *flakySave) saveDataAsChunk(util.FullPath) filer.SaveDataAsChunkFunctionType {
|
|
return func(reader io.Reader, name string, offset int64, tsNs int64, _ uint64) (*filer_pb.FileChunk, error) {
|
|
body, _ := io.ReadAll(reader)
|
|
s.bodies = append(s.bodies, string(body))
|
|
s.calls++
|
|
if s.onCall != nil {
|
|
s.onCall()
|
|
}
|
|
if s.calls <= len(s.failures) {
|
|
return nil, s.failures[s.calls-1]
|
|
}
|
|
return &filer_pb.FileChunk{FileId: "1,01", Offset: offset, Size: uint64(len(body)), ModifiedTsNs: tsNs}, nil
|
|
}
|
|
}
|
|
|
|
func newUploadTestPages(t *testing.T, writeback bool, save *flakySave) *ChunkedDirtyPages {
|
|
t.Helper()
|
|
wfs := newInvalidateTestWFS(t)
|
|
wfs.option.WritebackCache = writeback
|
|
fh := newFileHandle(wfs, 1, 2, &filer_pb.Entry{Name: "f", Attributes: &filer_pb.FuseAttributes{}})
|
|
fh.RememberPath("/dir/f")
|
|
pages := newMemoryChunkPages(fh, 1024)
|
|
t.Cleanup(pages.Destroy)
|
|
pages.saveDataAsChunk = save.saveDataAsChunk
|
|
return pages
|
|
}
|
|
|
|
var errVolumeRefused = fmt.Errorf("upload data: %w", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED})
|
|
|
|
// filerProxyOutage is what a filer proxy sends when it cannot reach the
|
|
// volume server: a 500 with an empty body.
|
|
func filerProxyOutage(t *testing.T) error {
|
|
t.Helper()
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
}))
|
|
defer server.Close()
|
|
_, err := operation.NewUploaderWithHttpClient(server.Client()).UploadData(context.Background(), []byte("hello"),
|
|
&operation.UploadOption{UploadUrl: server.URL, MaxAttempts: 1})
|
|
if err == nil {
|
|
t.Fatal("upload to the failing proxy succeeded")
|
|
}
|
|
return fmt.Errorf("upload data: %w", err)
|
|
}
|
|
|
|
// close() has already returned for a writeback handle, so a volume outage must
|
|
// not drop the buffered chunk: the same bytes go out again once it ends.
|
|
func TestChunkedDirtyPagesWritebackRetriesTransientUploadError(t *testing.T) {
|
|
for name, outage := range map[string]error{
|
|
"volume refused": errVolumeRefused,
|
|
"filer proxy 500": filerProxyOutage(t),
|
|
// With no volume server left the assign keeps timing out, and the
|
|
// filer client gives up after its own retries.
|
|
"assign timed out": fmt.Errorf("upload data: filerGrpcAddress assign volume: %w",
|
|
status.Error(codes.Unavailable, "filer 127.0.0.1:18888: call timed out")),
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
save := &flakySave{failures: []error{outage}}
|
|
pages := newUploadTestPages(t, true, save)
|
|
|
|
pages.saveChunkedFileIntervalToStorage(util.NewBytesReader([]byte("hello")), 0, 5, 1, func() {})
|
|
|
|
if err := pages.LastError(); err != nil {
|
|
t.Fatalf("LastError = %v, want the retried upload to succeed", err)
|
|
}
|
|
if save.calls != 2 || save.bodies[1] != "hello" {
|
|
t.Fatalf("uploads = %d with bodies %q, want a second upload of the same bytes", save.calls, save.bodies)
|
|
}
|
|
if chunks := pages.fh.GetEntry().GetChunks(); len(chunks) != 1 || chunks[0].Size != 5 {
|
|
t.Fatalf("entry chunks = %v, want the retried chunk", chunks)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestChunkedDirtyPagesUploadErrorWithoutRetry(t *testing.T) {
|
|
permanent := errors.New("upload data: assign volume failure: no writable volumes")
|
|
readOnly := fmt.Errorf("upload data: assign volume failure path:\"/unavailable/f\": %w",
|
|
filer_pb.AssignVolumeResponseError(&filer_pb.AssignVolumeResponse{
|
|
Error: "assign volume: read only: /unavailable",
|
|
ErrorCode: filer_pb.FilerError_READ_ONLY,
|
|
}))
|
|
tests := []struct {
|
|
name string
|
|
writeback bool
|
|
err error
|
|
discard func(pages *ChunkedDirtyPages)
|
|
wantErr error
|
|
}{
|
|
{name: "sync mode reports the outage to close", err: errVolumeRefused},
|
|
{name: "writeback permanent error", writeback: true, err: permanent},
|
|
{name: "writeback read-only path", writeback: true, err: readOnly},
|
|
{name: "writeback pages discarded mid-upload", writeback: true, err: errVolumeRefused,
|
|
discard: func(pages *ChunkedDirtyPages) { pages.Destroy() }},
|
|
{name: "writeback pages discarded during the retry wait", writeback: true, err: errVolumeRefused,
|
|
discard: func(pages *ChunkedDirtyPages) { time.AfterFunc(100*time.Millisecond, pages.Destroy) },
|
|
wantErr: errDirtyPagesDiscarded},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
save := &flakySave{failures: []error{tt.err}}
|
|
pages := newUploadTestPages(t, tt.writeback, save)
|
|
if tt.discard != nil {
|
|
save.onCall = func() { tt.discard(pages) }
|
|
}
|
|
wantErr := tt.err
|
|
if tt.wantErr != nil {
|
|
wantErr = tt.wantErr
|
|
}
|
|
|
|
pages.saveChunkedFileIntervalToStorage(util.NewBytesReader([]byte("hello")), 0, 5, 1, func() {})
|
|
|
|
if save.calls != 1 {
|
|
t.Fatalf("uploads = %d, want 1", save.calls)
|
|
}
|
|
if err := pages.LastError(); !errors.Is(err, wantErr) {
|
|
t.Fatalf("LastError = %v, want %v", err, wantErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A chunk whose pages were discarded while it uploaded must not reappear in
|
|
// the handle's entry, e.g. after a truncate replaced the content.
|
|
func TestChunkedDirtyPagesDiscardedUploadNotAttached(t *testing.T) {
|
|
save := &flakySave{}
|
|
pages := newUploadTestPages(t, true, save)
|
|
save.onCall = pages.Destroy
|
|
|
|
pages.saveChunkedFileIntervalToStorage(util.NewBytesReader([]byte("hello")), 0, 5, 1, func() {})
|
|
|
|
if chunks := pages.fh.GetEntry().GetChunks(); len(chunks) != 0 {
|
|
t.Fatalf("entry chunks = %v, want none from discarded pages", chunks)
|
|
}
|
|
}
|