diff --git a/test/fuse_integration/framework_test.go b/test/fuse_integration/framework_test.go index a7662e96e0..776e7f4647 100644 --- a/test/fuse_integration/framework_test.go +++ b/test/fuse_integration/framework_test.go @@ -29,6 +29,7 @@ type FuseTestFramework struct { mountProcess *managedProcess filerAddr string filerPort int + masterPort int weedBinary string isSetup bool config *TestConfig @@ -74,6 +75,10 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework { require.NoError(t, err) filerPort := freePort(t) + masterPort := freePort(t) + for masterPort == filerPort { + masterPort = freePort(t) + } return &FuseTestFramework{ t: t, @@ -83,6 +88,7 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework { logDir: filepath.Join(tempDir, "logs"), filerPort: filerPort, filerAddr: fmt.Sprintf("127.0.0.1:%d", filerPort), + masterPort: masterPort, weedBinary: findWeedBinary(), isSetup: false, config: config, @@ -218,6 +224,11 @@ func (f *FuseTestFramework) GetFilerAddr() string { return f.filerAddr } +// GetMasterAddr returns the master HTTP address of the mini cluster. +func (f *FuseTestFramework) GetMasterAddr() string { + return fmt.Sprintf("127.0.0.1:%d", f.masterPort) +} + // managedProcess is a started weed sub-command whose exit is watched, so a wait // for it to come up ends the moment it dies instead of burning its full timeout. type managedProcess struct { @@ -321,7 +332,7 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error { args = append(args, "-v=4") } adminPort := freePort(f.t) - for adminPort == f.filerPort { + for adminPort == f.filerPort || adminPort == f.masterPort { adminPort = freePort(f.t) } args = append(args, @@ -330,6 +341,8 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error { "-ip=127.0.0.1", "-ip.bind=127.0.0.1", "-filer.port="+strconv.Itoa(f.filerPort), + "-master.port="+strconv.Itoa(f.masterPort), + "-master.port.grpc="+strconv.Itoa(f.masterPort+grpcPortOffset), "-s3=false", "-webdav=false", "-admin.ui=false", diff --git a/test/fuse_integration/merge_read_view_test.go b/test/fuse_integration/merge_read_view_test.go new file mode 100644 index 0000000000..4ee8200dbd --- /dev/null +++ b/test/fuse_integration/merge_read_view_test.go @@ -0,0 +1,207 @@ +//go:build linux + +package fuse_test + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +const ( + mergePageSize = 8192 + mergePages = 32 +) + +// mergeHTTP bounds every HTTP call the test makes to the cluster. +var mergeHTTP = &http.Client{Timeout: 30 * time.Second} + +// TestReadOpenHandleAfterChunkMerge covers a file a flush merged while a +// descriptor kept it open. The merge rewrites the file into new chunks and the +// filer deletes the chunks it replaced, so after a vacuum only the new chunks +// exist. Reads through the open handle must find them at once, whether or not +// the handle has a write it has not flushed yet. +func TestReadOpenHandleAfterChunkMerge(t *testing.T) { + config := DefaultTestConfig() + // The client chunk cache could still hold a replaced chunk; the reads + // below have to reach the volume server. + config.CacheSizeMB = 0 + config.MountOptions = append(config.MountOptions, "-cacheCapacityMB=0") + framework := NewFuseTestFramework(t, config) + defer framework.Cleanup() + require.NoError(t, framework.Setup(config)) + + t.Run("CleanHandle", func(t *testing.T) { + testReadOpenHandleAfterChunkMerge(t, framework, "merged_clean.dat", false) + }) + t.Run("HandleWithUnflushedWrite", func(t *testing.T) { + testReadOpenHandleAfterChunkMerge(t, framework, "merged_dirty.dat", true) + }) +} + +func testReadOpenHandleAfterChunkMerge(t *testing.T, framework *FuseTestFramework, name string, unflushedWrite bool) { + path := filepath.Join(framework.GetMountPoint(), name) + held, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) + require.NoError(t, err) + defer held.Close() + + version := make([]int, mergePages) + whole := make([]byte, 0, mergePages*mergePageSize) + for p := 0; p < mergePages; p++ { + whole = append(whole, mergePagePattern(p, 0)...) + } + _, err = held.WriteAt(whole, 0) + require.NoError(t, err) + require.NoError(t, held.Sync()) + + // Overlapping 16-page windows: each leaves part of every older window + // visible, so the stored chunks soon add up to more than twice the file + // size and the next flush merges them. + var replaced []mergeChunk + for i, start := range []int{0, 3, 6, 9, 12, 15} { + before := mergeEntryChunks(t, framework, name) + window := make([]byte, 0, 16*mergePageSize) + for p := start; p < start+16; p++ { + window = append(window, mergePagePattern(p, i+1)...) + version[p] = i + 1 + } + _, err = held.WriteAt(window, int64(start*mergePageSize)) + require.NoError(t, err) + require.NoError(t, held.Sync()) + if after := mergeEntryChunks(t, framework, name); len(after) == 1 && after[0].ModifiedTsNs == 0 { + replaced = before + break + } + } + require.NotEmpty(t, replaced, "the overlapping writes never made a flush merge the chunks") + + // The filer deletes the replaced chunks after the merge commits; vacuum + // once they are deleted so that only the merged chunk is left on disk. + master := framework.GetMasterAddr() + require.Eventually(t, func() bool { + for _, c := range replaced { + if mergeChunkStatus(master, c, false) != http.StatusNotFound { + return false + } + } + return true + }, 30*time.Second, 200*time.Millisecond, "the replaced chunks were never deleted") + require.Eventually(t, func() bool { + resp, err := mergeHTTP.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=0.0001", master)) + if err != nil { + return false + } + resp.Body.Close() + for _, c := range replaced { + if mergeChunkStatus(master, c, true) != http.StatusNotFound { + return false + } + } + return true + }, 60*time.Second, time.Second, "vacuum left the replaced chunks on disk") + + if unflushedWrite { + _, err = held.WriteAt(mergePagePattern(mergePages-1, 99), int64((mergePages-1)*mergePageSize)) + require.NoError(t, err) + version[mergePages-1] = 99 + } + + // A second descriptor shares the handle the first one keeps open. Drop its + // pages from the kernel cache so every read goes to the mount. + reader, err := os.Open(path) + require.NoError(t, err) + defer reader.Close() + require.NoError(t, unix.Fadvise(int(reader.Fd()), 0, 0, unix.FADV_DONTNEED)) + started := time.Now() + buf := make([]byte, mergePageSize) + for p := 0; p < mergePages; p++ { + n, err := reader.ReadAt(buf, int64(p*mergePageSize)) + require.NoError(t, err, "read page %d through the open handle after the merge", p) + require.Equal(t, mergePageSize, n, "page %d", p) + require.Equal(t, mergePagePattern(p, version[p]), buf, "page %d", p) + } + assert.Less(t, time.Since(started), 5*time.Second, "reads through the open handle waited on chunks the merge replaced") +} + +func mergePagePattern(page, version int) []byte { + b := make([]byte, mergePageSize) + for i := 0; i < len(b); i += 8 { + binary.LittleEndian.PutUint32(b[i:], uint32(page)) + binary.LittleEndian.PutUint32(b[i+4:], uint32(version)) + } + return b +} + +type mergeChunk struct { + FileId string `json:"file_id"` + ModifiedTsNs int64 `json:"modified_ts_ns"` + Fid *struct { + VolumeId uint32 `json:"volume_id"` + FileKey uint64 `json:"file_key"` + Cookie uint32 `json:"cookie"` + } `json:"fid"` +} + +func (c mergeChunk) fileId() string { + if c.FileId != "" || c.Fid == nil { + return c.FileId + } + return fmt.Sprintf("%d,%x%08x", c.Fid.VolumeId, c.Fid.FileKey, c.Fid.Cookie) +} + +// mergeEntryChunks returns the chunks the filer stores for the file. +func mergeEntryChunks(t *testing.T, framework *FuseTestFramework, name string) []mergeChunk { + t.Helper() + resp, err := mergeHTTP.Get(fmt.Sprintf("http://%s/%s?metadata=true", framework.GetFilerAddr(), url.PathEscape(name))) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode, "filer metadata of %s", name) + var entry struct { + Chunks []mergeChunk `json:"chunks"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&entry)) + return entry.Chunks +} + +// mergeChunkStatus HEADs a chunk on the volume server holding it, and returns +// 0 when it cannot ask. With readDeleted, a deleted needle still answers until +// a vacuum removes it. +func mergeChunkStatus(master string, c mergeChunk, readDeleted bool) int { + fid := c.fileId() + volumeId := strings.SplitN(fid, ",", 2)[0] + resp, err := mergeHTTP.Get(fmt.Sprintf("http://%s/dir/lookup?volumeId=%s", master, volumeId)) + if err != nil { + return 0 + } + var lookup struct { + Locations []struct { + Url string `json:"url"` + } `json:"locations"` + } + err = json.NewDecoder(resp.Body).Decode(&lookup) + resp.Body.Close() + if err != nil || len(lookup.Locations) == 0 { + return 0 + } + target := fmt.Sprintf("http://%s/%s", lookup.Locations[0].Url, fid) + if readDeleted { + target += "?readDeleted=true" + } + head, err := mergeHTTP.Head(target) + if err != nil { + return 0 + } + head.Body.Close() + return head.StatusCode +} diff --git a/weed/mount/dirty_pages_chunked.go b/weed/mount/dirty_pages_chunked.go index 26a8d2aee6..3d97589f19 100644 --- a/weed/mount/dirty_pages_chunked.go +++ b/weed/mount/dirty_pages_chunked.go @@ -71,6 +71,10 @@ func (pages *ChunkedDirtyPages) HasWrites() bool { return pages.hasWrites } +func (pages *ChunkedDirtyPages) HasPendingUploads() bool { + return pages.hasWrites && pages.uploadPipeline.HasPendingUploads() +} + func (pages *ChunkedDirtyPages) FlushData() error { if !pages.hasWrites { return nil diff --git a/weed/mount/filehandle.go b/weed/mount/filehandle.go index e323e3f15c..8663e264cb 100644 --- a/weed/mount/filehandle.go +++ b/weed/mount/filehandle.go @@ -243,6 +243,40 @@ func (fh *FileHandle) rebaseEntry(merged, sent *filer_pb.Entry) { }) } +// installRewrittenChunks replaces the chunks a merge rewrote, taken from +// snapshot, with rewritten, and moves the read view to the result: the filer +// deletes the replaced chunks once the merge is committed, and a view still +// naming them would send reads to needles a vacuum removes. Chunks attached +// after snapshot was taken were not part of the merge and stay. +func (fh *FileHandle) installRewrittenChunks(snapshot, rewritten []*filer_pb.FileChunk) { + fh.UpdateEntry(func(e *filer_pb.Entry) { + chunks := withChunksAttachedSince(snapshot, rewritten, e.Chunks) + e.Chunks = chunks + err := fh.entryChunkGroup.SetChunks(chunks) + if err != nil { + glog.Warningf("merge %s: resolve chunks: %v", fh.FullPath(), err) + } + fh.chunkViewStale.Store(err != nil) + }) +} + +// withChunksAttachedSince returns processed, the chunks a flush made from +// snapshot, followed by the chunks uploads attached to current after snapshot +// was taken: they attach without the handle lock. +func withChunksAttachedSince(snapshot, processed, current []*filer_pb.FileChunk) []*filer_pb.FileChunk { + inSnapshot := make(map[string]bool, len(snapshot)) + for _, c := range snapshot { + inSnapshot[contentFileIdKey(c)] = true + } + chunks := append([]*filer_pb.FileChunk{}, processed...) + for _, c := range current { + if !inSnapshot[contentFileIdKey(c)] { + chunks = append(chunks, c) + } + } + return chunks +} + // refreshChunkView retries a read view whose manifest resolution failed. func (fh *FileHandle) refreshChunkView() { if !fh.chunkViewStale.Load() { diff --git a/weed/mount/flush_attach_race_test.go b/weed/mount/flush_attach_race_test.go new file mode 100644 index 0000000000..05f6e50c19 --- /dev/null +++ b/weed/mount/flush_attach_race_test.go @@ -0,0 +1,207 @@ +//go:build !freebsd + +package mount + +import ( + "bytes" + "context" + "errors" + "io" + "sync" + "testing" + "time" + + "github.com/seaweedfs/go-fuse/v2/fuse" + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" + "google.golang.org/protobuf/proto" +) + +// attachOnAssignPeer runs onAssign when the flush asks for a volume, which a +// handle with many small chunks does while it folds them into a manifest. +type attachOnAssignPeer struct { + *g6ConditionalPeer + once sync.Once + onAssign func() +} + +func (p *attachOnAssignPeer) AssignVolume(context.Context, *filer_pb.AssignVolumeRequest) (*filer_pb.AssignVolumeResponse, error) { + p.once.Do(p.onAssign) + return &filer_pb.AssignVolumeResponse{Error: "no volume in this test"}, nil +} + +// An upload attaches its chunk without the handle lock, so it can land while a +// flush is between taking its chunk snapshot and installing the compacted +// list. The chunk must stay on the handle and reach the filer. +func TestFlushKeepsChunkAttachedDuringCommit(t *testing.T) { + base := attributeAppendEntry(256) + base.CommitTsNs = 100 + peer := &g6ConditionalPeer{entry: proto.Clone(base).(*filer_pb.Entry)} + w, handle := g6AppendHandle(t, peer, base, 1000) + fh := w.GetHandle(FileHandleId(handle)) + + late := &filer_pb.FileChunk{Fid: &filer_pb.FileId{VolumeId: 9, FileKey: 1, Cookie: 1}, Offset: 0, Size: 256, ModifiedTsNs: 500} + startFakeFiler(t, w, &attachOnAssignPeer{g6ConditionalPeer: peer, onAssign: func() { + fh.AddChunks([]*filer_pb.FileChunk{late}) + fh.entryChunkGroup.AddChunk(late) + }}) + + if n, code := w.Write(nil, &fuse.WriteIn{InHeader: fuse.InHeader{NodeId: 42}, Fh: handle, Offset: 256 * 256, Size: 256}, bytes.Repeat([]byte{'W'}, 256)); n != 256 || code != fuse.OK { + t.Fatalf("write: %d %v", n, code) + } + if code := w.Flush(nil, &fuse.FlushIn{InHeader: fuse.InHeader{NodeId: 42}, Fh: handle}); code != fuse.OK { + t.Fatalf("flush: %v", code) + } + + has := func(chunks []*filer_pb.FileChunk) bool { + for _, c := range chunks { + if contentFileIdKey(c) == contentFileIdKey(late) { + return true + } + } + return false + } + if !has(fh.GetEntry().GetChunks()) { + t.Fatal("the handle lost the chunk an upload attached during the flush") + } + peer.mu.Lock() + defer peer.mu.Unlock() + if !has(peer.entry.Chunks) { + t.Fatal("the flush did not publish the chunk an upload attached during it") + } +} + +// Uploads keep attaching chunks while a flush clones its request entry, so the +// clone must hold the entry lock. Run with -race. +func TestFlushClonesRequestWhileUploadsAttach(t *testing.T) { + base := attributeAppendEntry(1) + base.CommitTsNs = 100 + peer := &g6ConditionalPeer{entry: proto.Clone(base).(*filer_pb.Entry)} + w, handle := g6AppendHandle(t, peer, base, 1000) + fh := w.GetHandle(FileHandleId(handle)) + + attached := make(chan struct{}) + go func() { + defer close(attached) + for key := uint64(1); key <= 2000; key++ { + fh.AddChunks([]*filer_pb.FileChunk{{Fid: &filer_pb.FileId{VolumeId: 9, FileKey: key, Cookie: 1}, Offset: 0, Size: 256, ModifiedTsNs: int64(key)}}) + time.Sleep(20 * time.Microsecond) + } + }() + for { + if err := w.flushMetadataToFiler(context.Background(), fh, "/dir", "file", 0, 0, nil); err != nil { + t.Fatalf("flush: %v", err) + } + select { + case <-attached: + return + default: + } + } +} + +// bloatedHandleWithPendingUpload opens a handle over chunks bloated enough to +// merge and starts uploading a write past them. The upload blocks until +// release is closed; drained reports its result. +func bloatedHandleWithPendingUpload(t *testing.T) (w *WFS, fh *FileHandle, peer *g6ConditionalPeer, base *filer_pb.Entry, release chan struct{}, drained chan error) { + t.Helper() + base = attributeAppendEntry(1) + base.Chunks = nil + for i := 0; i < 8; i++ { + base.Chunks = append(base.Chunks, &filer_pb.FileChunk{Fid: &filer_pb.FileId{VolumeId: 1, FileKey: uint64(i + 1), Cookie: 1}, Offset: int64(i * 64), Size: 256, ModifiedTsNs: int64(i + 1)}) + } + base.Attributes.FileSize = 7*64 + 256 + base.CommitTsNs = 100 + if _, _, bloated := shouldMergeChunks(base.Chunks, nil); !bloated { + t.Fatal("the test entry is not bloated enough to merge") + } + peer = &g6ConditionalPeer{entry: proto.Clone(base).(*filer_pb.Entry)} + w, handle := g6AppendHandle(t, peer, base, 1000) + fh = w.GetHandle(FileHandleId(handle)) + + started, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + fh.dirtyPages.randomWriter.(*ChunkedDirtyPages).saveDataAsChunk = func(util.FullPath) filer.SaveDataAsChunkFunctionType { + return func(r io.Reader, _ string, offset, ts int64, _ uint64) (*filer_pb.FileChunk, error) { + once.Do(func() { close(started) }) + <-release + body, err := io.ReadAll(r) + if err != nil { + return nil, err + } + return &filer_pb.FileChunk{Fid: &filer_pb.FileId{VolumeId: 2, FileKey: 1, Cookie: 1}, Offset: offset, Size: uint64(len(body)), ModifiedTsNs: ts}, nil + } + } + if n, code := w.Write(nil, &fuse.WriteIn{InHeader: fuse.InHeader{NodeId: 42}, Fh: handle, Offset: base.Attributes.FileSize, Size: 256}, bytes.Repeat([]byte{'W'}, 256)); n != 256 || code != fuse.OK { + t.Fatalf("write: %d %v", n, code) + } + drained = make(chan error, 1) + go func() { drained <- fh.dirtyPages.FlushData() }() + <-started + return w, fh, peer, base, release, drained +} + +// A flush that finds its chunks bloated while an upload is still in flight +// leaves the merge to a later flush instead of waiting for the upload under +// the handle and publication locks: a writeback upload retries until storage +// recovers, and every operation on the handle would wait with it. +func TestFlushLeavesMergeWhileUploadPending(t *testing.T) { + w, fh, peer, base, release, drained := bloatedHandleWithPendingUpload(t) + + flushed := make(chan error, 1) + go func() { flushed <- w.flushMetadataToFiler(context.Background(), fh, "/dir", "file", 0, 0, nil) }() + select { + case err := <-flushed: + if err != nil { + t.Fatalf("flush: %v", err) + } + case <-time.After(5 * time.Second): + close(release) + <-flushed + t.Fatal("the flush waited for a pending upload under the handle lock") + } + close(release) + if err := <-drained; err != nil { + t.Fatalf("drain: %v", err) + } + + peer.mu.Lock() + defer peer.mu.Unlock() + if len(peer.entry.Chunks) != len(base.Chunks) { + t.Fatalf("published %d chunks, want the %d unmerged ones", len(peer.entry.Chunks), len(base.Chunks)) + } +} + +// A flush that publishes while an upload is pending must not adopt a peer's +// inline content: the upload attaches its chunk next to it, and reads stop at +// the inline bytes, so the completed write would never be read. +func TestFlushWithUploadPendingRefusesPeerInlineContent(t *testing.T) { + w, fh, peer, _, release, drained := bloatedHandleWithPendingUpload(t) + defer func() { + close(release) + if err := <-drained; err != nil { + t.Fatalf("drain: %v", err) + } + }() + + inline := bytes.Repeat([]byte{'P'}, int(filer.FileSize(fh.GetEntry().GetEntry()))) + peer.mu.Lock() + peer.entry.Chunks = nil + peer.entry.Content = inline + peer.entry.Attributes.FileSize = uint64(len(inline)) + peer.entry.CommitTsNs = 200 + peer.mu.Unlock() + + if err := w.flushMetadataToFiler(context.Background(), fh, "/dir", "file", 0, 0, nil); !errors.Is(err, filer_pb.ErrPreconditionFailed) { + t.Fatalf("flush over a peer's inline content with an upload pending: %v, want a refused merge", err) + } + if content := fh.GetEntry().GetEntry().GetContent(); len(content) != 0 { + t.Fatalf("the handle adopted %d bytes of the peer's inline content", len(content)) + } + peer.mu.Lock() + defer peer.mu.Unlock() + if !bytes.Equal(peer.entry.Content, inline) || len(peer.entry.Chunks) != 0 { + t.Fatal("the flush published over the peer's inline content") + } +} diff --git a/weed/mount/merge_quiet_pages_test.go b/weed/mount/merge_quiet_pages_test.go new file mode 100644 index 0000000000..cdbb92dca3 --- /dev/null +++ b/weed/mount/merge_quiet_pages_test.go @@ -0,0 +1,542 @@ +//go:build !freebsd + +package mount + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "math/rand/v2" + "net" + "net/http" + "net/http/httptest" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/seaweedfs/go-fuse/v2/fuse" + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" + "google.golang.org/protobuf/proto" +) + +// proxyVolume is the filer's chunk proxy for a mount with +// VolumeServerAccess=filerProxy: chunk reads and uploads go to the filer's +// HTTP port, and the file ids for uploads come from its AssignVolume. +type proxyVolume struct { + *g6ConditionalPeer + mu sync.Mutex + stored map[string][]byte + assigns int + commits int + merges int +} + +func (p *proxyVolume) AssignVolume(context.Context, *filer_pb.AssignVolumeRequest) (*filer_pb.AssignVolumeResponse, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.assigns++ + return &filer_pb.AssignVolumeResponse{ + FileId: fmt.Sprintf("9,%x00000001", p.assigns), + Count: 1, + Location: &filer_pb.Location{Url: "127.0.0.1:1", PublicUrl: "127.0.0.1:1"}, + }, nil +} + +// CreateEntry commits like the filer: the chunks the previous entry named and +// the new one does not are deleted with the commit, as a vacuum leaves them. +// These tests write no chunk manifests, so the lists name every chunk. +func (p *proxyVolume) CreateEntry(ctx context.Context, r *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) { + p.g6ConditionalPeer.mu.Lock() + var before []*filer_pb.FileChunk + if p.entry != nil { + before = p.entry.Chunks + } + p.g6ConditionalPeer.mu.Unlock() + resp, err := p.g6ConditionalPeer.CreateEntry(ctx, r) + if err != nil || resp.GetError() != "" { + return resp, err + } + inBefore := make(map[string]bool, len(before)) + for _, c := range before { + inBefore[c.GetFileIdString()] = true + } + kept := make(map[string]bool, len(r.GetEntry().GetChunks())) + merged := false + for _, c := range r.GetEntry().GetChunks() { + kept[c.GetFileIdString()] = true + // maybeMergeChunks stamps the chunks it writes 0. + merged = merged || c.GetModifiedTsNs() == 0 && !inBefore[c.GetFileIdString()] + } + p.mu.Lock() + defer p.mu.Unlock() + p.commits++ + if merged { + p.merges++ + } + for _, c := range before { + if !kept[c.GetFileIdString()] { + delete(p.stored, c.GetFileIdString()) + } + } + return resp, nil +} + +func (p *proxyVolume) ServeHTTP(w http.ResponseWriter, r *http.Request) { + fid := r.URL.Query().Get("proxyChunkId") + if r.Method == http.MethodGet || r.Method == http.MethodHead { + p.mu.Lock() + body, ok := p.stored[fid] + p.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(body)) + return + } + var data []byte + var err error + if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/") { + var part io.Reader + if mr, mrErr := r.MultipartReader(); mrErr != nil { + err = mrErr + } else if part, err = mr.NextPart(); err == nil { + data, err = io.ReadAll(part) + } + } else { + data, err = io.ReadAll(r.Body) + } + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + p.mu.Lock() + p.stored[fid] = data + p.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{"size": len(data), "eTag": "test"}) +} + +func (p *proxyVolume) counts() (commits, merges int) { + p.mu.Lock() + defer p.mu.Unlock() + return p.commits, p.merges +} + +// committed returns the filer's entry and the content its chunks hold, a +// newer chunk covering an older one. +func (p *proxyVolume) committed(t *testing.T) (*filer_pb.Entry, []byte) { + t.Helper() + p.g6ConditionalPeer.mu.Lock() + entry := proto.Clone(p.entry).(*filer_pb.Entry) + p.g6ConditionalPeer.mu.Unlock() + chunks := append([]*filer_pb.FileChunk(nil), entry.Chunks...) + sort.SliceStable(chunks, func(x, y int) bool { return chunks[x].ModifiedTsNs < chunks[y].ModifiedTsNs }) + content := make([]byte, entry.Attributes.FileSize) + p.mu.Lock() + defer p.mu.Unlock() + for _, c := range chunks { + body, ok := p.stored[c.GetFileIdString()] + if !ok || uint64(len(body)) != c.Size || c.Offset+int64(c.Size) > int64(len(content)) { + t.Fatalf("the filer names a chunk that is not stored as named: %v", c) + } + copy(content[c.Offset:], body) + } + return entry, content +} + +// startProxyFiler serves chunks with proxy and the filer API with filerServer. +// Its gRPC port is the HTTP port plus 10000, the default, so the plain +// host:port address names both: the chunk proxy builds upload URLs from the +// address string itself. +func startProxyFiler(t *testing.T, w *WFS, proxy http.Handler, filerServer filer_pb.SeaweedFilerServer) { + t.Helper() + for attempt := 0; ; attempt++ { + srv := httptest.NewServer(proxy) + _, portText, _ := net.SplitHostPort(srv.Listener.Addr().String()) + port, _ := strconv.Atoi(portText) + listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port+10000)) + if err != nil { + srv.Close() + if attempt < 50 { + continue + } + t.Fatalf("no free gRPC port next to an HTTP port: %v", err) + } + t.Cleanup(srv.Close) + server := pb.NewGrpcServer() + filer_pb.RegisterSeaweedFilerServer(server, filerServer) + go server.Serve(listener) + t.Cleanup(server.Stop) + w.option.FilerAddresses = []pb.ServerAddress{pb.ServerAddress(srv.Listener.Addr().String())} + return + } +} + +// openProxyHandle opens /dir/file, which the filer vol holds as base, on a +// mount whose chunk requests go to proxy. Uploads run four at a time. +func openProxyHandle(t *testing.T, vol *proxyVolume, base *filer_pb.Entry, proxy http.Handler) (*WFS, *FileHandle, uint64) { + t.Helper() + w := newInvalidateTestWFS(t) + w.option.CacheDirForWrite = t.TempDir() + w.option.DefaultPermissions = true + w.posixLocks = NewPosixLockTable() + w.option.ConcurrentWriters = 4 + w.concurrentWriters = util.NewLimitedConcurrentExecutor(4) + vol.g6ConditionalPeer = &g6ConditionalPeer{entry: proto.Clone(base).(*filer_pb.Entry)} + startProxyFiler(t, w, proxy, vol) + w.inodeToPath.Lookup("/dir", 1, true, false, 0, false) + w.inodeToPath.Lookup("/dir/file", 1, false, false, 42, true) + w.inodeToPath.MarkChildrenCached("/dir") + if err := w.metaCache.InsertEntry(context.Background(), filer.FromPbEntry("/dir", proto.Clone(base).(*filer_pb.Entry)), 1); err != nil { + t.Fatal(err) + } + out := &fuse.OpenOut{} + if code := w.Open(nil, &fuse.OpenIn{InHeader: fuse.InHeader{NodeId: 42}, Flags: syscall.O_RDWR}, out); code != fuse.OK { + t.Fatal(code) + } + fh := w.GetHandle(FileHandleId(out.Fh)) + t.Cleanup(fh.dirtyPages.Destroy) + return w, fh, out.Fh +} + +// readView reads size bytes through the handle's read view, as a read that +// misses the dirty pages does. +func readView(t *testing.T, fh *FileHandle, size int) []byte { + t.Helper() + buf := make([]byte, size) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + n, _, err := fh.entryChunkGroup.ReadDataAt(ctx, int64(size), buf, 0) + if err != nil && err != io.EOF { + t.Fatalf("read through the open handle: %v", err) + } + return buf[:n] +} + +// A merge replaces every chunk it reads, while uploads attach their chunks +// without the handle lock, so a flush merges only when no upload is pending. +// When it does merge, reads through the open handle must follow it: the +// commit deletes the replaced chunks. +func TestFlushMergeThroughChunkProxy(t *testing.T) { + payload := func(seed uint32) []byte { + b := make([]byte, 256) + x := seed * 2654435761 + for i := range b { + x = x*1103515245 + 12345 + b[i] = byte(x >> 16) + } + return b + } + chunk := func(key uint64, offset int64) *filer_pb.FileChunk { + return &filer_pb.FileChunk{Fid: &filer_pb.FileId{VolumeId: 1, FileKey: key, Cookie: 1}, Offset: offset, Size: 256, ModifiedTsNs: int64(key)} + } + // Three overlapping 256-byte chunks hold a 300-byte file: 768 bytes + // stored, more than twice the content. + a, b, c := chunk(1, 0), chunk(2, 22), chunk(3, 44) + pa, pb, pc := payload(1), payload(2), payload(3) + base := &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{ + Inode: 42, Uid: 2000, Gid: 20, FileMode: 0644, FileSize: 300, Mtime: 10, + }, Chunks: []*filer_pb.FileChunk{a, b, c}, CommitTsNs: 100} + content := append(append(append([]byte{}, pa[:22]...), pb[:22]...), pc...) + newVolume := func() *proxyVolume { + return &proxyVolume{stored: map[string][]byte{ + a.GetFileIdString(): pa, b.GetFileIdString(): pb, c.GetFileIdString(): pc, + }} + } + // flush publishes as doFlush does after its own drain. + flush := func(t *testing.T, w *WFS, fh *FileHandle, publication *contentPublication) { + t.Helper() + if err := w.flushMetadataToFiler(context.Background(), fh, "/dir", "file", 0, 0, nil, publication); err != nil { + t.Fatalf("flush: %v", err) + } + } + + t.Run("Quiet", func(t *testing.T) { + vol := newVolume() + w, fh, _ := openProxyHandle(t, vol, base, vol) + flush(t, w, fh, &contentPublication{writes: fh.pageWrites.Load()}) + + entry, committed := vol.committed(t) + if len(entry.Chunks) != 1 || entry.Chunks[0].ModifiedTsNs != 0 { + t.Fatalf("the quiet flush did not merge: %v", entry.Chunks) + } + vol.mu.Lock() + blobs := len(vol.stored) + vol.mu.Unlock() + if blobs != 1 || !bytes.Equal(committed, content) { + t.Fatalf("after the merge %d chunks are stored, want only the merged one, with the file's content", blobs) + } + if got := readView(t, fh, len(content)); !bytes.Equal(got, content) { + t.Fatalf("read %d bytes through the open handle after the merge, not the merged content", len(got)) + } + }) + + // An older page is still uploading when a newer page of the same bytes + // has attached. A merge would read the newer page, and the older one, + // attaching after it, would outrank the merged chunks, which are stamped 0. + t.Run("UploadInFlight", func(t *testing.T) { + vol := newVolume() + var hold atomic.Bool + uploading, release := make(chan struct{}), make(chan struct{}) + w, fh, handle := openProxyHandle(t, vol, base, http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead && hold.CompareAndSwap(true, false) { + close(uploading) + <-release + } + vol.ServeHTTP(rw, r) + })) + // Let a held upload go however the test ends: the proxy's cleanup, + // which runs after this one, waits for its handlers. + var released sync.Once + releaseUpload := func() { released.Do(func() { close(release) }) } + t.Cleanup(releaseUpload) + await := func(done <-chan struct{}, what string) { + t.Helper() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for %s", what) + } + } + write := func(data []byte) { + t.Helper() + if n, code := w.Write(nil, &fuse.WriteIn{InHeader: fuse.InHeader{NodeId: 42}, Fh: handle, Offset: 100, Size: uint32(len(data))}, data); int(n) != len(data) || code != fuse.OK { + t.Fatalf("write: %d %v", n, code) + } + } + pipeline := fh.dirtyPages.randomWriter.(*ChunkedDirtyPages).uploadPipeline + drained := make(chan struct{}, 2) + drain := func() { pipeline.FlushAll(); drained <- struct{}{} } + + // The flush's own drain ran before both writes. + publication := &contentPublication{writes: fh.pageWrites.Load()} + // Another flush drains the older page, whose upload hangs. + write(bytes.Repeat([]byte{'o'}, 40)) + hold.Store(true) + go drain() + await(uploading, "the older page to upload") + // A third drains the newer page, which uploads and attaches. + newer := bytes.Repeat([]byte{'n'}, 40) + write(newer) + go drain() + for deadline := time.Now().Add(5 * time.Second); len(fh.GetEntry().GetChunks()) < len(base.Chunks)+1; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatal("the newer page did not attach") + } + } + flush(t, w, fh, publication) + releaseUpload() + await(drained, "the drains") + await(drained, "the drains") + + want := append([]byte{}, content...) + copy(want[100:], newer) + written := func(b []byte) []byte { + if len(b) < 140 { + return b + } + return b[100:140] + } + if got := readView(t, fh, len(want)); !bytes.Equal(got, want) { + t.Fatalf("read %q at [100,140) through the open handle, want the newer page", written(got)) + } + if _, merges := vol.counts(); merges != 0 { + t.Fatal("the flush merged while an upload was in flight") + } + // With no upload left, the next flush merges. + flush(t, w, fh, &contentPublication{writes: fh.pageWrites.Load()}) + if _, merges := vol.counts(); merges != 1 { + t.Fatalf("%d merges once the uploads were done, want 1", merges) + } + if _, committed := vol.committed(t); !bytes.Equal(committed, want) { + t.Fatalf("the filer holds %q at [100,140), want the newer page", written(committed)) + } + if got := readView(t, fh, len(want)); !bytes.Equal(got, want) { + t.Fatalf("read %q at [100,140) through the open handle after the merge, want the newer page", written(got)) + } + }) +} + +// Writes, flushes and reads run together on one handle while overwrites keep +// its chunks bloated, so flushes merge when no upload is pending and leave the +// merge to a later one otherwise. Uploads take a random while, +// and each commit deletes the chunks it replaced. Every read must succeed +// promptly, and in the end the handle and the filer must both hold exactly +// what was written. +func TestFlushMergeRacesWithWritesAndReads(t *testing.T) { + const writers, region, writesPerWriter = 4, 2048, 100 + size := writers * region + + // Each writer owns one region, which starts as one stored chunk. + want := make([]byte, size) + base := &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{ + Inode: 42, Uid: 2000, Gid: 20, FileMode: 0644, FileSize: uint64(size), Mtime: 10, + }, CommitTsNs: 100} + vol := &proxyVolume{stored: map[string][]byte{}} + for i := range writers { + c := &filer_pb.FileChunk{Fid: &filer_pb.FileId{VolumeId: 1, FileKey: uint64(i + 1), Cookie: 1}, Offset: int64(i * region), Size: region, ModifiedTsNs: int64(i + 1)} + body := want[i*region : (i+1)*region] + for j := range body { + body[j] = byte(rand.Uint32()) + } + vol.stored[c.GetFileIdString()] = append([]byte(nil), body...) + base.Chunks = append(base.Chunks, c) + } + w, fh, handle := openProxyHandle(t, vol, base, http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + time.Sleep(rand.N(2 * time.Millisecond)) + } + vol.ServeHTTP(rw, r) + })) + header := fuse.InHeader{NodeId: 42} + + // doFlush's steps: record the page count, drain the pages, publish if + // dirty. doFlush reads the handle's flags without the handle lock, so + // this drains through the pipeline and reads them under the lock. + pages := fh.dirtyPages.randomWriter.(*ChunkedDirtyPages) + flush := func() error { + publication := &contentPublication{writes: fh.pageWrites.Load()} + pages.uploadPipeline.FlushAll() + if err := pages.LastError(); err != nil { + return err + } + lock := w.fhLockTable.AcquireLock("test flush", fh.fh, util.SharedLock) + dirty := fh.dirtyMetadata + w.fhLockTable.ReleaseLock(fh.fh, lock) + if !dirty { + return nil + } + return w.flushMetadataToFiler(context.Background(), fh, "/dir", "file", 0, 0, nil, publication) + } + read := func(offset, n int) ([]byte, fuse.Status, time.Duration) { + buf := make([]byte, n) + start := time.Now() + result, code := w.Read(nil, &fuse.ReadIn{InHeader: header, Fh: handle, Offset: uint64(offset), Size: uint32(n)}, buf) + if code != fuse.OK { + return nil, code, time.Since(start) + } + data, code := result.Bytes(buf) + return data, code, time.Since(start) + } + + // One write, flush and read first, so the first upload and the first + // chunk read do not race each other through the clients' lazy setup. + first := []byte("first write") + if n, code := w.Write(nil, &fuse.WriteIn{InHeader: header, Fh: handle, Size: uint32(len(first))}, first); code != fuse.OK || int(n) != len(first) { + t.Fatalf("first write: %d %v", n, code) + } + copy(want, first) + if err := flush(); err != nil { + t.Fatalf("first flush: %v", err) + } + if got, code, _ := read(0, size); code != fuse.OK || !bytes.Equal(got, want) { + t.Fatalf("first read: %v", code) + } + + stop := make(chan struct{}) + stopped := func() bool { + select { + case <-stop: + return true + default: + return false + } + } + var background sync.WaitGroup + // Several flushers, so one can drain while another publishes. + for range 3 { + background.Go(func() { + for !stopped() { + if err := flush(); err != nil { + t.Errorf("flush: %v", err) + return + } + } + }) + } + background.Go(func() { + for !stopped() { + offset := rand.IntN(size) + n := 1 + rand.IntN(1024) + if offset+n > size { + n = size - offset + } + if _, code, took := read(offset, n); code != fuse.OK || took > 2*time.Second { + t.Errorf("read [%d,%d): %v after %v", offset, offset+n, code, took) + return + } + } + }) + // Now and then the writers pause for a few commits, so the uploads finish + // and a flush merges. + var pause sync.RWMutex + background.Go(func() { + for !stopped() { + time.Sleep(rand.N(10 * time.Millisecond)) + pause.Lock() + commits, merges := vol.counts() + for deadline := time.Now().Add(100 * time.Millisecond); time.Now().Before(deadline); time.Sleep(time.Millisecond) { + if c, m := vol.counts(); c >= commits+4 || m > merges { + break + } + } + pause.Unlock() + } + }) + + var writes sync.WaitGroup + for i := range writers { + writes.Go(func() { + mine := want[i*region : (i+1)*region] + for range writesPerWriter { + n := 1 + rand.IntN(region/2) + at := rand.IntN(region - n + 1) + data := make([]byte, n) + for j := range data { + data[j] = byte(rand.Uint32()) + } + pause.RLock() + written, code := w.Write(nil, &fuse.WriteIn{InHeader: header, Fh: handle, Offset: uint64(i*region + at), Size: uint32(n)}, data) + pause.RUnlock() + if code != fuse.OK || int(written) != n { + t.Errorf("write [%d,%d): %d %v", i*region+at, i*region+at+n, written, code) + return + } + copy(mine[at:], data) + } + }) + } + writes.Wait() + _, mergedWhileWriting := vol.counts() + close(stop) + background.Wait() + if t.Failed() { + return + } + if err := flush(); err != nil { + t.Fatalf("last flush: %v", err) + } + commits, merges := vol.counts() + t.Logf("%d commits, %d merges while writing, %d in all", commits, mergedWhileWriting, merges) + if mergedWhileWriting == 0 { + t.Fatal("no flush merged while the writes ran") + } + if got, code, _ := read(0, size); code != fuse.OK || !bytes.Equal(got, want) { + t.Fatalf("read back through the handle: %v, %d bytes, as written: %v", code, len(got), bytes.Equal(got, want)) + } + if _, committed := vol.committed(t); !bytes.Equal(committed, want) { + t.Fatal("the filer's entry does not hold what was written") + } +} diff --git a/weed/mount/merge_read_view_test.go b/weed/mount/merge_read_view_test.go new file mode 100644 index 0000000000..6e277dae60 --- /dev/null +++ b/weed/mount/merge_read_view_test.go @@ -0,0 +1,97 @@ +package mount + +import ( + "context" + "errors" + "fmt" + "io" + "reflect" + "syscall" + "testing" + "time" + + "github.com/seaweedfs/go-fuse/v2/fuse" + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// A chunk merge rewrites the whole file into new chunks, and the filer deletes +// the chunks it replaced once the merge is committed. A handle that stays open +// across the merge must read the new chunks: after a vacuum the replaced ones +// are gone, and a read view still naming them sends reads to missing needles. +func TestMergeMovesReadViewOffReplacedChunks(t *testing.T) { + wfs := newInvalidateTestWFS(t) + inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false) + + chunk := func(volumeId uint32, key uint64, offset int64, size int, ts int64) *filer_pb.FileChunk { + return &filer_pb.FileChunk{Fid: &filer_pb.FileId{VolumeId: volumeId, FileKey: key, Cookie: 1}, Offset: offset, Size: uint64(size), ModifiedTsNs: ts} + } + // A bloated file: b rewrote the tail of a. + a := chunk(1, 1, 0, 8, 1) + b := chunk(1, 2, 4, 8, 2) + // The merge's rewrite, stamped zero as maybeMergeChunks stamps it. + merged := chunk(2, 1, 0, 12, 0) + // A write that was attached while the merge ran. + late := chunk(3, 1, 10, 4, 3) + + entry := &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{Inode: 42, FileMode: 0644, FileSize: 12}, Chunks: []*filer_pb.FileChunk{a, b}} + fh, _, _, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, entry, 0, 0) + snapshot := fh.GetEntry().GetChunks() + + // Only the merge's chunk and the late write are still stored: a and b went + // with the merge's commit and a vacuum. + cache := newFakeChunkCache() + cache.Put(merged.GetFileIdString(), []byte("aaaabbbbbbbb")) + cache.Put(late.GetFileIdString(), []byte("dddd")) + group, err := filer.NewChunkGroup(func(ctx context.Context, fileId string) ([]string, error) { + return nil, errors.New("needle " + fileId + " was vacuumed") + }, cache, []*filer_pb.FileChunk{a, b}, 1, nil, nil) + if err != nil { + t.Fatal(err) + } + fh.entryChunkGroup = group + fh.AddChunks([]*filer_pb.FileChunk{late}) + if err := fh.entryChunkGroup.AddChunk(late); err != nil { + t.Fatal(err) + } + + buf := make([]byte, 14) + if _, _, err := fh.entryChunkGroup.ReadDataAt(context.Background(), int64(len(buf)), buf, 0); err == nil || errors.Is(err, io.EOF) { + t.Fatal("a read through the replaced chunks succeeded; the test would not notice a stale read view") + } + + fh.installRewrittenChunks(snapshot, []*filer_pb.FileChunk{merged}) + + var got []string + for _, c := range fh.GetEntry().GetChunks() { + got = append(got, c.GetFileIdString()) + } + if want := []string{merged.GetFileIdString(), late.GetFileIdString()}; !reflect.DeepEqual(got, want) { + t.Fatalf("chunks after the merge: %v, want the rewrite and the late write %v", got, want) + } + + n, _, err := fh.entryChunkGroup.ReadDataAt(context.Background(), int64(len(buf)), buf, 0) + if err != nil && !errors.Is(err, io.EOF) { + t.Fatalf("read through the open handle after the merge: %v", err) + } + if got := string(buf[:n]); got != "aaaabbbbbbdddd" { + t.Fatalf("read %q after the merge, want %q", got, "aaaabbbbbbdddd") + } +} + +// An upload that fails while a flush drains the pages before a merge is a +// write error: it maps the way doFlush maps its own FlushData failure, and the +// metadata flush does not retry it as a filer error. +func TestPageUploadErrorIsAWriteError(t *testing.T) { + full := &pageUploadError{err: fmt.Errorf("flush data: %w", errors.New("assign volume: no free volumes left"))} + if got := flushErrorToFuseStatus(fmt.Errorf("flush /dir/file: %w", full)); got != fuse.Status(syscall.ENOSPC) { + t.Fatalf("a full cluster during the drain maps to %v, want ENOSPC", got) + } + if !isPageUploadError(fmt.Errorf("flush /dir/file: %w", full)) { + t.Fatal("a wrapped upload failure is not recognized, so the metadata flush would retry it") + } + if got := flushErrorToFuseStatus(errors.New("fh flush create /dir/file: refused")); got != fuse.EIO { + t.Fatalf("other flush failures map to %v, want EIO as before", got) + } +} diff --git a/weed/mount/page_writer.go b/weed/mount/page_writer.go index 5f690a30bd..3d3b85f9e5 100644 --- a/weed/mount/page_writer.go +++ b/weed/mount/page_writer.go @@ -57,6 +57,10 @@ func (pw *PageWriter) HasWrites() bool { return pw.randomWriter.HasWrites() } +func (pw *PageWriter) HasPendingUploads() bool { + return pw.randomWriter.HasPendingUploads() +} + func (pw *PageWriter) ReadDirtyDataAt(data []byte, offset int64, tsNs int64) (maxStop int64) { glog.V(4).Infof("ReadDirtyDataAt %v [%d, %d)", pw.fh.inode, offset, offset+int64(len(data))) diff --git a/weed/mount/page_writer/dirty_pages.go b/weed/mount/page_writer/dirty_pages.go index 7d94d3c0ab..42ad62831f 100644 --- a/weed/mount/page_writer/dirty_pages.go +++ b/weed/mount/page_writer/dirty_pages.go @@ -4,6 +4,7 @@ type DirtyPages interface { AddPage(offset int64, data []byte, isSequential bool, tsNs int64) (n int, err error) FlushData() error HasWrites() bool + HasPendingUploads() bool ReadDirtyDataAt(data []byte, startOffset int64, tsNs int64) (maxStop int64) Destroy() LockForRead(startOffset, stopOffset int64) diff --git a/weed/mount/page_writer/upload_pipeline.go b/weed/mount/page_writer/upload_pipeline.go index 4762f1b1c8..e40acab62a 100644 --- a/weed/mount/page_writer/upload_pipeline.go +++ b/weed/mount/page_writer/upload_pipeline.go @@ -205,6 +205,14 @@ func (up *UploadPipeline) FlushAll() { up.waitForCurrentWritersToComplete() } +// HasPendingUploads reports whether written data is still waiting to be +// uploaded and attached. +func (up *UploadPipeline) HasPendingUploads() bool { + up.chunksLock.Lock() + defer up.chunksLock.Unlock() + return len(up.writableChunks) > 0 || atomic.LoadInt32(&up.uploaderCount) > 0 +} + func (up *UploadPipeline) flushChunks() { up.chunksLock.Lock() defer up.chunksLock.Unlock() diff --git a/weed/mount/weedfs_file_sync.go b/weed/mount/weedfs_file_sync.go index da68a4ca7a..cf22c111cb 100644 --- a/weed/mount/weedfs_file_sync.go +++ b/weed/mount/weedfs_file_sync.go @@ -226,7 +226,7 @@ func (wfs *WFS) doFlush(ctx context.Context, fh *FileHandle, uid, gid uint32, al if err := retryMetadataFlushIf(ctx, func() error { return wfs.flushMetadataToFiler(ctx, fh, dir, name, uid, gid, commit, publication) }, func(err error) bool { - return !commit.spent() && isRetryableFilerError(err) + return !commit.spent() && !isPageUploadError(err) && isRetryableFilerError(err) }, func(nextAttempt, totalAttempts int, backoff time.Duration, err error) { glog.Warningf("%v fh %d flush: retrying metadata flush (attempt %d/%d) after %v: %v", fileFullPath, fh.fh, nextAttempt, totalAttempts, backoff, err) @@ -235,7 +235,7 @@ func (wfs *WFS) doFlush(ctx context.Context, fh *FileHandle, uid, gid uint32, al publication.retire(err) } glog.Errorf("%v fh %d flush: %v", fileFullPath, fh.fh, err) - return grpcErrorToFuseStatus(err) + return flushErrorToFuseStatus(err) } if IsDebugFileReadWrite { @@ -245,6 +245,28 @@ func (wfs *WFS) doFlush(ctx context.Context, fh *FileHandle, uid, gid uint32, al return fuse.OK } +// pageUploadError is a page upload failure a metadata flush found before it +// published anything. doFlush reports it as a write error, as it does when its +// own FlushData fails, instead of retrying it as a filer error. +type pageUploadError struct{ err error } + +func (e *pageUploadError) Error() string { return e.err.Error() } +func (e *pageUploadError) Unwrap() error { return e.err } + +func isPageUploadError(err error) bool { + var upload *pageUploadError + return errors.As(err, &upload) +} + +// flushErrorToFuseStatus maps a failed metadata flush to a FUSE status. +func flushErrorToFuseStatus(err error) fuse.Status { + var upload *pageUploadError + if errors.As(err, &upload) { + return writeErrorToFuseStatus(upload.err) + } + return grpcErrorToFuseStatus(err) +} + // flushMetadataToFiler sends the file's chunk references and attributes to the filer. // This is shared between the synchronous doFlush path and the async flush completion. // @@ -320,19 +342,48 @@ func (wfs *WFS) flushMetadataToFiler(ctx context.Context, fh *FileHandle, dir, n glog.V(4).Infof("%s set chunks: %v", fileFullPath, len(entry.GetChunks())) - manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(entry.GetChunks()) + snapshot := entry.GetChunks() + manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(snapshot) chunks, _, err := filer.CompactFileChunks(context.Background(), wfs.LookupFn(), nonManifestChunks) if err != nil { return fmt.Errorf("compact chunks of %s: %w", fileFullPath, err) } - if mergedChunks, mergeErr := wfs.maybeMergeChunks(fileFullPath, chunks, manifestChunks); mergeErr != nil { + deferMerge := false + if _, _, bloated := shouldMergeChunks(chunks, manifestChunks); bloated { + // The merge replaces every chunk it reads. Uploads attach their chunks + // without the handle lock, and writes that landed after doFlush drained + // the pages may still be uploading. A writeback upload retries until + // storage recovers, so leave the merge to a later flush rather than + // wait for it under these locks. With none pending, the lock holds + // writes off and nothing attaches during the merge: take the chunks + // again. A failed upload fails the flush as a write error, before + // anything claims the bytes it did not store. + if fh.dirtyPages.HasPendingUploads() { + deferMerge = true + } else { + if err := fh.dirtyPages.FlushData(); err != nil { + return &pageUploadError{err: err} + } + snapshot = entry.GetChunks() + manifestChunks, nonManifestChunks = filer.SeparateManifestChunks(snapshot) + if chunks, _, err = filer.CompactFileChunks(context.Background(), wfs.LookupFn(), nonManifestChunks); err != nil { + return fmt.Errorf("compact chunks of %s: %w", fileFullPath, err) + } + } + } + + rewrote := false + if deferMerge { + glog.V(3).Infof("%s: uploads pending, leaving the chunk merge to a later flush", fileFullPath) + } else if mergedChunks, mergeErr := wfs.maybeMergeChunks(fileFullPath, chunks, manifestChunks); mergeErr != nil { glog.V(0).Infof("maybeMergeChunks %s: %v", fileFullPath, mergeErr) } else if mergedChunks != nil { chunks = mergedChunks manifestChunks = nil fh.noteRewrite() + rewrote = true } // no chunk deleter here: a failed fold reports the blobs it saved @@ -341,13 +392,21 @@ func (wfs *WFS) flushMetadataToFiler(ctx context.Context, fh *FileHandle, dir, n // not good, but should be ok glog.V(0).Infof("MaybeManifestize: %v", manifestErr) } - entry.Chunks = append(chunks, manifestChunks...) + if rewrote { + fh.installRewrittenChunks(snapshot, append(chunks, manifestChunks...)) + } else { + fh.UpdateEntry(func(e *filer_pb.Entry) { + e.Chunks = withChunksAttachedSince(snapshot, append(chunks, manifestChunks...), e.Chunks) + }) + } // Clone the proto entry for the filer request so that mapPbIdFromLocalToFiler // does not mutate the file handle's live entry. Without the clone, a concurrent // Lookup can observe filer-side uid/gid on the file handle entry and return it // to the kernel, which caches it and then rejects opens by the local user. - requestEntry := proto.Clone(entry.GetEntry()).(*filer_pb.Entry) + // Uploads attach chunks without the handle lock, so clone under the entry lock. + var requestEntry *filer_pb.Entry + fh.UpdateEntry(func(e *filer_pb.Entry) { requestEntry = proto.Clone(e).(*filer_pb.Entry) }) request := &filer_pb.CreateEntryRequest{ Directory: string(dir), Entry: requestEntry, @@ -362,8 +421,11 @@ func (wfs *WFS) flushMetadataToFiler(ctx context.Context, fh *FileHandle, dir, n // The chunk merge and the manifest above read and upload chunks, which can // take longer than the whole commit budget. It is charged only from here. + // Writes that landed before the lock may still be uploading, so a merge onto a + // moved entry must keep room for them. + buffered := fh.dirtyPages.HasPendingUploads() commitCtx, end := commit.begin(ctx) - resp, merged, err := wfs.createContentEntry(commitCtx, fh, request, baseSnapshot, false, publication) + resp, merged, err := wfs.createContentEntry(commitCtx, fh, request, baseSnapshot, buffered, publication) end() if err != nil { glog.Errorf("fh flush create %s: %v", fileFullPath, err)