mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-16 00:38:53 +01:00
* mount: fail the write when the swap file chunk cannot be created #7957 added a nil check after NewSwapFileChunk so a failed swap file creation returns an error instead of panicking. The result was assigned to pageChunk, a PageChunk interface, before the check, and an interface holding a nil *SwapFileChunk is not nil, so the check never fired: SaveDataAt stored the nil chunk and panicked in WriteDataAt. Check the *SwapFileChunk before converting it. The new test points the swap directory below a regular file; on master it panics in SwapFileChunk.WriteDataAt. Signed-off-by: anant <170962632+antcybersec@users.noreply.github.com> * mount: grow the file only after its write is buffered Write set the new size, times, pending append offset and uncommitted bytes before AddPage. When AddPage failed (now that a swap file chunk that cannot be created returns an error), the write was reported as failed but the larger size stayed on the handle's entry, so the next flush published it with a hole where the data should be. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: report a partly buffered write as a short write PageWriter.AddPage fills one chunk at a time, so when a later chunk cannot be created the earlier chunks already hold their bytes. Return how much was buffered, and let Write grow the size and times over that prefix and report it, instead of failing the whole write while the prefix still reaches the next flush. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: report the bytes a failed copy_file_range already buffered A streaming copy whose page fails partway returned an error even though earlier pages, or the start of the failing one, were already buffered and would reach the next flush with the destination size left short. Count the buffered bytes, grow the destination over them, and return them as a short copy. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: free the copy test's destination pages The buffered memory chunk counts toward the process-wide memory chunk ceiling, so repeated runs of the test pushed later ones onto the unusable swap directory. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: count only the bytes a swap chunk stored A swap chunk whose file write fails stores a prefix or nothing, but SaveDataAt returned no error and AddPage counted the whole span as buffered. SaveDataAt now reports the short write, and AddPage passes the stored count up so a write is never acknowledged past it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * mount: fail a partly buffered writeback write Kernel writeback ignores the count in a WRITE reply, so a short write there drops the unbuffered rest without an error. A write flagged as page-cache writeback now keeps its buffered prefix but returns the error, which the kernel reports on fsync. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Signed-off-by: anant <170962632+antcybersec@users.noreply.github.com> Co-authored-by: anant <170962632+antcybersec@users.noreply.github.com> Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
277 lines
9.8 KiB
Go
277 lines
9.8 KiB
Go
package mount
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/go-fuse/v2/fuse"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
|
)
|
|
|
|
// slowDataFiler answers AssignVolume only after assignDelay, as a filer does
|
|
// while its master is slow or no volume slot is free, and points every
|
|
// assignment at a volume stub. CreateEntry, the metadata commit, answers at
|
|
// once; with stuckCommit it holds the request until the caller gives up, like
|
|
// a filer wedged on its store.
|
|
type slowDataFiler struct {
|
|
fakeFilerServer
|
|
volume string
|
|
assignDelay time.Duration
|
|
stuckCommit bool
|
|
assigns atomic.Int32
|
|
commits atomic.Int32
|
|
committedChunks atomic.Int32
|
|
committedManifests atomic.Int32
|
|
}
|
|
|
|
func (s *slowDataFiler) AssignVolume(ctx context.Context, req *filer_pb.AssignVolumeRequest) (*filer_pb.AssignVolumeResponse, error) {
|
|
n := s.assigns.Add(1)
|
|
select {
|
|
case <-time.After(s.assignDelay):
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
return &filer_pb.AssignVolumeResponse{
|
|
FileId: fmt.Sprintf("3,%02x637037d6", n),
|
|
Count: 1,
|
|
Location: &filer_pb.Location{Url: s.volume, PublicUrl: s.volume},
|
|
}, nil
|
|
}
|
|
|
|
func (s *slowDataFiler) CreateEntry(ctx context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
|
|
s.commits.Add(1)
|
|
chunks := req.GetEntry().GetChunks()
|
|
manifests, _ := filer.SeparateManifestChunks(chunks)
|
|
s.committedChunks.Store(int32(len(chunks)))
|
|
s.committedManifests.Store(int32(len(manifests)))
|
|
if s.stuckCommit {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
}
|
|
return s.fakeFilerServer.CreateEntry(ctx, req)
|
|
}
|
|
|
|
// newVolumeStub stores chunk uploads the way a volume server answers them.
|
|
func newVolumeStub(t *testing.T) string {
|
|
t.Helper()
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
reader, err := r.MultipartReader()
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
part, err := reader.NextPart()
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
data, err := io.ReadAll(part)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
w.Header().Set("Content-MD5", part.Header.Get("Content-MD5"))
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusCreated)
|
|
fmt.Fprintf(w, `{"size":%d}`, len(data))
|
|
}))
|
|
t.Cleanup(server.Close)
|
|
return strings.TrimPrefix(server.URL, "http://")
|
|
}
|
|
|
|
// newSlowDataFlush opens /dir/file with dirty data not yet uploaded, written
|
|
// after the chunks the file already holds (stored), on a mount whose filer
|
|
// takes assignDelay to assign a chunk, and shortens the commit deadline to
|
|
// deadline.
|
|
func newSlowDataFlush(t *testing.T, assignDelay, deadline time.Duration, stuckCommit bool, stored ...*filer_pb.FileChunk) (*WFS, *FileHandle, *slowDataFiler) {
|
|
t.Helper()
|
|
|
|
saved := metadataFlushTimeout
|
|
metadataFlushTimeout = deadline
|
|
t.Cleanup(func() { metadataFlushTimeout = saved })
|
|
|
|
wfs := newInvalidateTestWFS(t)
|
|
wfs.option.ConcurrentWriters = 4
|
|
wfs.option.uniqueCacheDirForWrite = t.TempDir()
|
|
wfs.concurrentWriters = util.NewLimitedConcurrentExecutor(4)
|
|
|
|
fake := &slowDataFiler{volume: newVolumeStub(t), assignDelay: assignDelay, stuckCommit: stuckCommit}
|
|
startFakeFiler(t, wfs, fake)
|
|
// Upload straight to the assigned volume server, as mounts do by default.
|
|
wfs.option.VolumeServerAccess = "direct"
|
|
wfs.filerClient = wdclient.NewFilerClient(wfs.option.FilerAddresses, wfs.option.GrpcDialOption, "")
|
|
|
|
data := []byte("the last shard of a checkpoint, written just before close()")
|
|
offset := int64(filer.TotalSize(stored))
|
|
inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false)
|
|
fh, _, _, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, &filer_pb.Entry{
|
|
Name: "file",
|
|
Attributes: &filer_pb.FuseAttributes{FileSize: uint64(offset) + uint64(len(data)), FileMode: 0100644},
|
|
Chunks: stored,
|
|
}, 0, 0)
|
|
t.Cleanup(func() { wfs.ReleaseHandle(fh.fh) })
|
|
if _, err := fh.dirtyPages.AddPage(offset, data, true, time.Now().UnixNano()); err != nil {
|
|
t.Fatalf("AddPage: %v", err)
|
|
}
|
|
fh.dirtyMetadata = true
|
|
return wfs, fh, fake
|
|
}
|
|
|
|
// storedSmallChunks returns n chunks a file already holds, back to back from
|
|
// offset 0, each small enough to count toward a manifest of small chunks.
|
|
func storedSmallChunks(n int) []*filer_pb.FileChunk {
|
|
const size = 1024
|
|
chunks := make([]*filer_pb.FileChunk, n)
|
|
for i := range chunks {
|
|
chunks[i] = &filer_pb.FileChunk{
|
|
FileId: fmt.Sprintf("4,%02x637037d6", i+1),
|
|
Offset: int64(i) * size,
|
|
Size: size,
|
|
ModifiedTsNs: int64(i + 1),
|
|
}
|
|
}
|
|
return chunks
|
|
}
|
|
|
|
// checkCommitted asserts the flush made the given number of uploads, then
|
|
// committed once, an entry of one chunk.
|
|
func checkCommitted(t *testing.T, fh *FileHandle, fake *slowDataFiler, uploads int32) {
|
|
t.Helper()
|
|
if got := fake.assigns.Load(); got != uploads {
|
|
t.Fatalf("AssignVolume calls = %d, want %d", got, uploads)
|
|
}
|
|
if got := fake.commits.Load(); got != 1 {
|
|
t.Fatalf("CreateEntry calls = %d, want 1", got)
|
|
}
|
|
if got := fake.committedChunks.Load(); got != 1 {
|
|
t.Fatalf("committed chunks = %d, want 1", got)
|
|
}
|
|
if fh.dirtyMetadata {
|
|
t.Fatal("handle still dirty after the commit")
|
|
}
|
|
}
|
|
|
|
// close() after an upload that outlasts the commit deadline still commits:
|
|
// the deadline bounds CreateEntry, which the filer answers at once. Started at
|
|
// the top of Flush, the deadline was spent by the slow assign, and close()
|
|
// returned EIO for data that was already stored.
|
|
func TestFlushCommitsAfterSlowUpload(t *testing.T) {
|
|
wfs, fh, fake := newSlowDataFlush(t, time.Second, 200*time.Millisecond, false)
|
|
|
|
status := wfs.Flush(nil, &fuse.FlushIn{InHeader: fuse.InHeader{NodeId: fh.inode}, Fh: uint64(fh.fh)})
|
|
if status != fuse.OK {
|
|
t.Fatalf("Flush = %v, want OK: the data is stored and the filer answers the commit at once", status)
|
|
}
|
|
checkCommitted(t, fh, fake, 1)
|
|
}
|
|
|
|
// fsync() gets the same commit-only deadline as close().
|
|
func TestFsyncCommitsAfterSlowUpload(t *testing.T) {
|
|
wfs, fh, fake := newSlowDataFlush(t, time.Second, 200*time.Millisecond, false)
|
|
|
|
status := wfs.Fsync(nil, &fuse.FsyncIn{InHeader: fuse.InHeader{NodeId: fh.inode}, Fh: uint64(fh.fh)})
|
|
if status != fuse.OK {
|
|
t.Fatalf("Fsync = %v, want OK", status)
|
|
}
|
|
checkCommitted(t, fh, fake, 1)
|
|
}
|
|
|
|
// A truncate below dirty data flushes the data first (see SetAttr); that
|
|
// flush gets the commit-only deadline too.
|
|
func TestTruncateFlushCommitsAfterSlowUpload(t *testing.T) {
|
|
wfs, fh, fake := newSlowDataFlush(t, time.Second, 200*time.Millisecond, false)
|
|
|
|
in := &fuse.SetAttrIn{}
|
|
in.NodeId = fh.inode
|
|
in.Valid = fuse.FATTR_SIZE
|
|
in.Size = 8
|
|
var out fuse.AttrOut
|
|
if status := wfs.SetAttr(nil, in, &out); status != fuse.OK {
|
|
t.Fatalf("SetAttr size down = %v, want OK", status)
|
|
}
|
|
if got := fake.commits.Load(); got != 1 {
|
|
t.Fatalf("CreateEntry calls = %d, want 1", got)
|
|
}
|
|
if out.Size != 8 {
|
|
t.Fatalf("size after truncate = %d, want 8", out.Size)
|
|
}
|
|
}
|
|
|
|
// The data upload is not the only one a flush makes. One that completes a
|
|
// batch of small chunks folds them into a manifest and uploads that too,
|
|
// while it prepares the entry it is about to commit. That upload must not
|
|
// spend the commit deadline either.
|
|
func TestFlushCommitsAfterSlowManifestUpload(t *testing.T) {
|
|
// One chunk short of a batch: the chunk this flush uploads completes it.
|
|
stored := storedSmallChunks(filer.SmallChunkManifestBatch - 1)
|
|
wfs, fh, fake := newSlowDataFlush(t, time.Second, 200*time.Millisecond, false, stored...)
|
|
|
|
status := wfs.Flush(nil, &fuse.FlushIn{InHeader: fuse.InHeader{NodeId: fh.inode}, Fh: uint64(fh.fh)})
|
|
if status != fuse.OK {
|
|
t.Fatalf("Flush = %v, want OK: the data and the manifest are stored and the filer answers the commit at once", status)
|
|
}
|
|
checkCommitted(t, fh, fake, 2) // the data chunk, then the manifest
|
|
if got := fake.committedManifests.Load(); got != 1 {
|
|
t.Fatalf("committed manifests = %d, want 1: this flush no longer uploads a manifest", got)
|
|
}
|
|
}
|
|
|
|
// The deadline still bounds the commit itself: against a filer that never
|
|
// answers CreateEntry, close() fails once the deadline passes instead of
|
|
// hanging, and the handle stays dirty for Release to try again.
|
|
func TestFlushCommitDeadlineStillBoundsAStuckFiler(t *testing.T) {
|
|
const deadline = 200 * time.Millisecond
|
|
wfs, fh, fake := newSlowDataFlush(t, 0, deadline, true)
|
|
|
|
start := time.Now()
|
|
status := wfs.Flush(nil, &fuse.FlushIn{InHeader: fuse.InHeader{NodeId: fh.inode}, Fh: uint64(fh.fh)})
|
|
elapsed := time.Since(start)
|
|
if status == fuse.OK {
|
|
t.Fatal("Flush = OK against a filer that never committed")
|
|
}
|
|
if elapsed < deadline || elapsed > deadline+5*time.Second {
|
|
t.Fatalf("Flush returned after %v, want about the %v commit deadline", elapsed, deadline)
|
|
}
|
|
if got := fake.commits.Load(); got != 1 {
|
|
t.Fatalf("CreateEntry calls = %d, want 1: no retry once the deadline passed", got)
|
|
}
|
|
if !fh.dirtyMetadata {
|
|
t.Fatal("handle marked clean after a failed commit")
|
|
}
|
|
}
|
|
|
|
// The attempts of one flush share the budget: what one filer call used is gone
|
|
// for the next. A nil budget bounds nothing.
|
|
func TestCommitBudgetIsSharedByItsCalls(t *testing.T) {
|
|
budget := &commitBudget{left: 50 * time.Millisecond}
|
|
ctx, end := budget.begin(context.Background())
|
|
<-ctx.Done() // a filer that never answers
|
|
end()
|
|
if !budget.spent() {
|
|
t.Fatalf("%v left after a call used the whole budget", budget.left)
|
|
}
|
|
ctx, end = budget.begin(context.Background())
|
|
if ctx.Err() == nil {
|
|
t.Fatal("a call started with time left after the budget ran out")
|
|
}
|
|
end()
|
|
|
|
var none *commitBudget
|
|
ctx, end = none.begin(context.Background())
|
|
if _, bounded := ctx.Deadline(); bounded || none.spent() {
|
|
t.Fatal("a nil budget bounded the call")
|
|
}
|
|
end()
|
|
}
|