seaweedfs/weed/mount/weedfs_attr_cas_test.go

//go:build !freebsd

package mount

import (
	"context"
	"fmt"
	"runtime"
	"strings"
	"sync"
	"syscall"
	"testing"
	"time"

	"github.com/seaweedfs/go-fuse/v2/fuse"
	"github.com/seaweedfs/seaweedfs/weed/cluster"
	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"google.golang.org/protobuf/proto"
)

// The writer's acknowledged entry and a second mount's cached entry diverge
// before the attribute syscall. The filer implements the existing atomic
// IF_ENTRY_EQUAL contract; no sleeps or FUSE device are needed for this order.
type attributeCASFiler struct {
	filer_pb.UnimplementedSeaweedFilerServer
	mu                 sync.Mutex
	entry              *filer_pb.Entry
	updates, conflicts int
	beforeUpdate       func(int, *filer_pb.Entry)
	afterCommit        func()
	updateErr          error
	missing            bool
}

func (s *attributeCASFiler) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.missing {
		return nil, status.Error(codes.NotFound, "deleted")
	}
	return &filer_pb.LookupDirectoryEntryResponse{Entry: proto.Clone(s.entry).(*filer_pb.Entry)}, nil
}

func (s *attributeCASFiler) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.updates++
	if s.updateErr != nil {
		return nil, s.updateErr
	}
	if s.beforeUpdate != nil {
		s.beforeUpdate(s.updates, s.entry)
	}
	if clauses := req.GetCondition().GetClauses(); len(clauses) > 0 && !proto.Equal(clauses[0].GetExpectedEntry(), s.entry) {
		s.conflicts++
		return nil, status.Error(codes.FailedPrecondition, "entry changed")
	}
	s.entry = proto.Clone(req.Entry).(*filer_pb.Entry)
	if s.afterCommit != nil {
		// Release the owner lock: concurrent CreateEntry must be able to publish
		// while the committed UpdateEntry response is held at this barrier.
		s.mu.Unlock()
		s.afterCommit()
		s.mu.Lock()
	}
	return &filer_pb.UpdateEntryResponse{LogTsNs: 2000}, nil
}

func TestAttributeOnlyConflictOutcomes(t *testing.T) {
	for _, tc := range []struct {
		name    string
		flags   uint32
		prepare func(*attributeCASFiler)
		want    fuse.Status
		calls   int
	}{
		{"bounded", 0, func(s *attributeCASFiler) {
			s.beforeUpdate = func(n int, e *filer_pb.Entry) { e.Extended["concurrent"] = []byte(fmt.Sprint(n)) }
		}, fuse.Status(syscall.EAGAIN), 3},
		{"replacement", 0, func(s *attributeCASFiler) { s.entry.Attributes.Inode = 99 }, fuse.Status(syscall.ESTALE), 1},
		{"deleted", 0, func(s *attributeCASFiler) { s.missing = true }, fuse.ENOENT, 1},
		{"create-existing", xattr_CREATE, func(s *attributeCASFiler) { s.entry.Extended["xattr-user.test"] = []byte("other mount") }, fuse.Status(syscall.EEXIST), 1},
		{"replace-removed", xattr_REPLACE, func(s *attributeCASFiler) { delete(s.entry.Extended, "xattr-user.test") }, fuse.ENODATA, 1},
		{"permission", 0, func(s *attributeCASFiler) { s.updateErr = status.Error(codes.PermissionDenied, "refused") }, fuse.Status(syscall.EACCES), 1},
	} {
		t.Run(tc.name, func(t *testing.T) {
			wfs := newInvalidateTestWFS(t)
			cached := attributeAppendEntry(1)
			if tc.flags == xattr_REPLACE {
				cached.Extended["xattr-user.test"] = []byte("cached")
			}
			fake := &attributeCASFiler{entry: attributeAppendEntry(300)}
			tc.prepare(fake)
			startFakeFiler(t, wfs, fake)
			path := util.FullPath("/dir/file")
			wfs.inodeToPath.Lookup(util.FullPath("/dir"), 1, true, false, 0, false)
			wfs.inodeToPath.Lookup(path, 1, false, false, 42, true)
			wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir"))
			if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry("/dir", cached), 1); err != nil {
				t.Fatal(err)
			}
			in := &fuse.SetXAttrIn{Flags: tc.flags}
			in.NodeId = 42
			if code := wfs.SetXAttr(nil, in, "user.test", []byte("changed")); code != tc.want {
				t.Fatalf("got %v, want %v", code, tc.want)
			}
			fake.mu.Lock()
			defer fake.mu.Unlock()
			if fake.updates != tc.calls {
				t.Fatalf("updates=%d want %d", fake.updates, tc.calls)
			}
			if len(fake.entry.Chunks) != 300 || fake.entry.Attributes.FileSize != 76800 {
				t.Fatal("failed attribute op changed content")
			}
		})
	}
}

func (s *attributeCASFiler) CreateEntry(ctx context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.entry = proto.Clone(req.Entry).(*filer_pb.Entry)
	return &filer_pb.CreateEntryResponse{LogTsNs: 2000}, nil
}

func attributeAppendEntry(records int) *filer_pb.Entry {
	e := &filer_pb.Entry{Name: "file", Attributes: &filer_pb.FuseAttributes{
		Inode: 42, Uid: 2000, Gid: 20, FileMode: 0644, FileSize: uint64(records * 256), Mtime: 10,
	}, Extended: map[string][]byte{"xattr-user.other": []byte("keep"), "xattr-user.remove": []byte("remove")}}
	for i := 0; i < records; i++ {
		e.Chunks = append(e.Chunks, &filer_pb.FileChunk{Fid: &filer_pb.FileId{VolumeId: 1, FileKey: uint64(i + 1), Cookie: 1}, Offset: int64(i * 256), Size: 256, ModifiedTsNs: int64(i + 1)})
	}
	return e
}

// A peer's acknowledged attribute change leaves the content alone, so our
// append's conditional flush must still publish, keeping the peer's change.
func TestAppendFlushAfterPeerAttributeChange(t *testing.T) {
	for _, op := range []string{"none", "chmod", "chown", "utimes", "setxattr"} {
		t.Run(op, func(t *testing.T) {
			writer, peer := newInvalidateTestWFS(t), newInvalidateTestWFS(t)
			peer.atimeMap = make(map[uint64]time.Time)
			base := attributeAppendEntry(1)
			base.CommitTsNs = 100
			fake := &contentFlushConditionFiler{entry: proto.Clone(base).(*filer_pb.Entry)}
			startFakeFiler(t, writer, fake)
			startFakeFiler(t, peer, fake)
			path := util.FullPath("/dir/file")
			local := proto.Clone(base).(*filer_pb.Entry)
			writer.mapPbIdFromFilerToLocal(local)
			fh := newFileHandle(writer, 1, 42, local)
			fh.RememberPath(path)
			t.Cleanup(fh.dirtyPages.Destroy)
			writer.fhMap.inode2fh[42], writer.fhMap.fh2inode[1] = fh, 42
			appended := attributeAppendEntry(2).Chunks[1]
			fh.UpdateEntry(func(e *filer_pb.Entry) {
				e.Chunks = append(e.Chunks, appended)
				e.Attributes.FileSize = 512
			})
			fh.dirtyMetadata = true

			peer.inodeToPath.Lookup("/dir", 1, true, false, 0, false)
			peer.inodeToPath.Lookup(path, 1, false, false, 42, true)
			peer.inodeToPath.MarkChildrenCached("/dir")
			if err := peer.metaCache.InsertEntry(context.Background(), filer.FromPbEntry("/dir", base), 1); err != nil {
				t.Fatal(err)
			}
			code := fuse.OK
			in := &fuse.SetAttrIn{}
			in.NodeId = 42
			switch op {
			case "chmod":
				in.Valid, in.Mode = fuse.FATTR_MODE, 0600
			case "chown":
				in.Valid, in.Owner.Uid, in.Owner.Gid = fuse.FATTR_UID|fuse.FATTR_GID, 1001, 21
			case "utimes":
				in.Valid, in.Mtime = fuse.FATTR_MTIME, 123
			case "setxattr":
				code = peer.SetXAttr(nil, &fuse.SetXAttrIn{InHeader: fuse.InHeader{NodeId: 42}}, "user.test", []byte("peer"))
			}
			if in.Valid != 0 {
				code = peer.SetAttr(nil, in, &fuse.AttrOut{})
			}
			if code != fuse.OK {
				t.Fatalf("peer %s: %v", op, code)
			}

			if code = writer.Fsync(nil, &fuse.FsyncIn{InHeader: fuse.InHeader{NodeId: 42}, Fh: 1}); code != fuse.OK {
				t.Fatalf("append fsync after peer %s: %v", op, code)
			}
			fake.mu.Lock()
			defer fake.mu.Unlock()
			got := fake.entry
			if len(got.Chunks) != 2 || got.Attributes.FileSize != 512 {
				t.Fatalf("append not published after peer %s: %v", op, got)
			}
			a := got.Attributes
			kept := map[string]bool{
				"none":     true,
				"chmod":    a.FileMode&0777 == 0600,
				"chown":    a.Uid == 1001 && a.Gid == 21,
				"utimes":   a.Mtime == 123,
				"setxattr": string(got.Extended["xattr-user.test"]) == "peer",
			}[op]
			if !kept {
				t.Fatalf("append flush dropped the peer's %s: %v", op, got)
			}
		})
	}
}

func TestAttributeOnlyOpsPreserveAcknowledgedAppends(t *testing.T) {
	for _, op := range []string{"chmod", "chown", "utimes", "setxattr", "removexattr"} {
		for _, stale := range []bool{false, true} {
			for _, open := range []bool{false, true} {
				t.Run(fmt.Sprintf("%s/stale=%t/open=%t", op, stale, open), func(t *testing.T) {
					wfs := newInvalidateTestWFS(t)
					wfs.atimeMap = make(map[uint64]time.Time)
					current := attributeAppendEntry(300)
					fake := &attributeCASFiler{entry: proto.Clone(current).(*filer_pb.Entry)}
					startFakeFiler(t, wfs, fake)
					cached := proto.Clone(current).(*filer_pb.Entry)
					if stale {
						cached = attributeAppendEntry(1)
						cached.Extended["xattr-user.other"] = []byte("obsolete")
						cached.Attributes.Gid = 17
					}
					path := util.FullPath("/dir/file")
					wfs.inodeToPath.Lookup(util.FullPath("/dir"), 1, true, false, 0, false)
					wfs.inodeToPath.Lookup(path, 1, false, false, 42, true)
					wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir"))
					if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry("/dir", cached), 1); err != nil {
						t.Fatal(err)
					}
					var fh *FileHandle
					if open {
						local := proto.Clone(cached).(*filer_pb.Entry)
						wfs.mapPbIdFromFilerToLocal(local)
						fh = &FileHandle{fh: 1, inode: 42, wfs: wfs, entry: &LockedEntry{Entry: local}}
						fh.dirtyPages = newPageWriter(fh, 1024)
						fh.RememberPath(path)
						wfs.fhMap.inode2fh[42], wfs.fhMap.fh2inode[1] = fh, 42
						t.Cleanup(func() { fh.dirtyPages.Destroy() })
					}
					var code fuse.Status
					out := &fuse.AttrOut{}
					if op == "removexattr" {
						code = wfs.RemoveXAttr(nil, &fuse.InHeader{NodeId: 42}, "user.remove")
					} else if op == "setxattr" {
						in := &fuse.SetXAttrIn{}
						in.NodeId = 42
						code = wfs.SetXAttr(nil, in, "user.test", []byte("new"))
					} else {
						in := &fuse.SetAttrIn{}
						in.NodeId = 42
						switch op {
						case "chmod":
							in.Valid, in.Mode = fuse.FATTR_MODE, 0600
						case "chown":
							in.Valid, in.Owner.Uid, in.Owner.Gid = fuse.FATTR_UID|fuse.FATTR_GID, 1001, 21
						case "utimes":
							in.Valid, in.Mtime, in.Mtimensec = fuse.FATTR_MTIME|fuse.FATTR_ATIME, 123, 456
						}
						code = wfs.SetAttr(nil, in, out)
					}
					if code != fuse.OK {
						t.Fatalf("attribute syscall: %v", code)
					}
					if (op == "chmod" || op == "utimes") && out.Attr.Uid != 1000 {
						t.Fatalf("reply uid %d, want the local id 1000", out.Attr.Uid)
					}
					if open {
						if code := wfs.doFlush(context.Background(), fh, 1000, 20, false, 0); code != fuse.OK {
							t.Fatalf("flush after attribute syscall: %v", code)
						}
					}
					fake.mu.Lock()
					defer fake.mu.Unlock()
					if len(fake.entry.Chunks) != 300 || fake.entry.Attributes.FileSize != 76800 {
						t.Fatalf("lost fsync-acknowledged appends: chunks=%d size=%d, want 300/76800", len(fake.entry.Chunks), fake.entry.Attributes.FileSize)
					}
					for i, chunk := range fake.entry.Chunks {
						if chunk.GetFileIdString() != current.Chunks[i].GetFileIdString() || chunk.Offset != int64(i*256) || chunk.Size != 256 {
							t.Fatalf("acknowledged record %d no longer references its original bytes: %v", i, chunk)
						}
					}
					switch op {
					case "chmod":
						if fake.entry.Attributes.FileMode != 0600 {
							t.Fatal("mode not saved")
						}
					case "chown":
						if fake.entry.Attributes.Uid != 1001 || fake.entry.Attributes.Gid != 21 {
							t.Fatal("owner not saved")
						}
					case "utimes":
						if fake.entry.Attributes.Mtime != 123 || fake.entry.Attributes.MtimeNs != 456 {
							t.Fatal("mtime not saved")
						}
					case "setxattr":
						if string(fake.entry.Extended["xattr-user.test"]) != "new" {
							t.Fatal("xattr not saved")
						}
					case "removexattr":
						if _, exists := fake.entry.Extended["xattr-user.remove"]; exists {
							t.Fatal("xattr not removed")
						}
					}
					if string(fake.entry.Extended["xattr-user.other"]) != "keep" {
						t.Fatal("unrelated xattr lost")
					}
					if op != "chown" && (fake.entry.Attributes.Uid != 2000 || fake.entry.Attributes.Gid != 20) {
						t.Fatal("unrelated owner or uid mapping lost")
					}
				})
			}
		}
	}
}

// The filer holds the attribute RPC while the real WRITE handler dirties the
// same clean handle. Its subsequent flush must keep both the acknowledged
// attribute and the buffered bytes. The upload adapter stores bytes in memory.
func TestAcknowledgedAttributeSurvivesWriteDuringRPC(t *testing.T) {
	for _, op := range []string{"chmod", "chown", "utimes", "setxattr", "removexattr"} {
		t.Run(op, func(t *testing.T) {
			wfs := newInvalidateTestWFS(t)
			wfs.atimeMap = make(map[uint64]time.Time)
			wfs.option.ConcurrentWriters = 1
			wfs.concurrentWriters = util.NewLimitedConcurrentExecutor(1)
			current := attributeAppendEntry(1)
			fake := &attributeCASFiler{entry: proto.Clone(current).(*filer_pb.Entry)}
			entered, resume := make(chan struct{}), make(chan struct{})
			var once sync.Once
			t.Cleanup(func() { once.Do(func() { close(resume) }) })
			fake.beforeUpdate = func(n int, e *filer_pb.Entry) {
				if n == 1 {
					close(entered)
					<-resume
				}
			}
			startFakeFiler(t, wfs, fake)
			path := util.FullPath("/dir/file")
			wfs.inodeToPath.Lookup(path, 1, false, false, 42, true)
			local := proto.Clone(current).(*filer_pb.Entry)
			wfs.mapPbIdFromFilerToLocal(local)
			fh := newFileHandle(wfs, 1, 42, local)
			fh.RememberPath(path)
			wfs.fhMap.inode2fh[42], wfs.fhMap.fh2inode[1] = fh, 42
			save := &flakySave{}
			fh.dirtyPages.randomWriter.(*ChunkedDirtyPages).saveDataAsChunk = save.saveDataAsChunk
			t.Cleanup(fh.dirtyPages.Destroy)
			done := make(chan fuse.Status, 1)
			go func() {
				if op == "setxattr" {
					done <- wfs.SetXAttr(nil, &fuse.SetXAttrIn{InHeader: fuse.InHeader{NodeId: 42}}, "user.test", []byte("acknowledged"))
				} else if op == "removexattr" {
					done <- wfs.RemoveXAttr(nil, &fuse.InHeader{NodeId: 42}, "user.remove")
				} else {
					in := &fuse.SetAttrIn{}
					in.NodeId = 42
					switch op {
					case "chmod":
						in.Valid, in.Mode = fuse.FATTR_MODE, 0600
					case "chown":
						in.Valid, in.Owner.Uid, in.Owner.Gid = fuse.FATTR_UID|fuse.FATTR_GID, 1001, 21
					case "utimes":
						in.Valid, in.Mtime, in.Mtimensec = fuse.FATTR_MTIME, 123, 456
					}
					done <- wfs.SetAttr(nil, in, &fuse.AttrOut{})
				}
			}()
			select {
			case <-entered:
			case <-time.After(10 * time.Second):
				t.Fatal("attribute RPC never arrived")
			}
			payload := []byte("background write while attribute RPC waits")
			in := &fuse.WriteIn{}
			in.NodeId, in.Fh, in.Offset, in.Size = 42, 1, 256, uint32(len(payload))
			if n, code := wfs.Write(nil, in, payload); code != fuse.OK || n != uint32(len(payload)) {
				t.Fatalf("WRITE: %d %v", n, code)
			}
			once.Do(func() { close(resume) })
			select {
			case code := <-done:
				if code != fuse.OK {
					t.Fatalf("attribute: %v", code)
				}
			case <-time.After(10 * time.Second):
				t.Fatal("attribute RPC did not finish")
			}
			if code := wfs.doFlush(context.Background(), fh, 1000, 20, false, 0); code != fuse.OK {
				t.Fatalf("flush: %v", code)
			}
			fake.mu.Lock()
			defer fake.mu.Unlock()
			switch op {
			case "chmod":
				if fake.entry.Attributes.FileMode != 0600 {
					t.Fatal("flush undid acknowledged chmod")
				}
			case "chown":
				if fake.entry.Attributes.Uid != 1001 || fake.entry.Attributes.Gid != 21 {
					t.Fatal("flush undid acknowledged chown")
				}
			case "utimes":
				if fake.entry.Attributes.Mtime != 123 || fake.entry.Attributes.MtimeNs != 456 {
					t.Fatal("flush undid acknowledged utimes")
				}
			case "setxattr":
				if string(fake.entry.Extended["xattr-user.test"]) != "acknowledged" {
					t.Fatal("flush undid acknowledged setxattr")
				}
			case "removexattr":
				if _, exists := fake.entry.Extended["xattr-user.remove"]; exists {
					t.Fatal("flush undid acknowledged removexattr")
				}
			}
			if len(save.bodies) != 1 || save.bodies[0] != string(payload) {
				t.Fatalf("pending bytes changed: %q", save.bodies)
			}
			if fake.entry.Attributes.FileSize != 256+uint64(len(payload)) || len(fake.entry.Chunks) != 2 {
				t.Fatalf("pending content lost: %v", fake.entry)
			}
			original, appended := false, false
			for _, chunk := range fake.entry.Chunks {
				switch chunk.Offset {
				case 0:
					original = chunk.GetFileIdString() == current.Chunks[0].GetFileIdString() && chunk.Size == 256
				case 256:
					appended = chunk.GetFileIdString() == "1,01" && chunk.Size == uint64(len(payload))
				}
			}
			if !original || !appended {
				t.Fatalf("chunk identity/range changed: %v", fake.entry.Chunks)
			}

		})
	}
}

// Force owner-commit -> concurrent WRITE -> whole-entry flush -> RPC response.
// The corrected caller must serialize publication across the entire interval.
func TestAttributePublicationBeforeResponse(t *testing.T) {
	for _, contention := range []string{"response", "installation", "retention"} {
		for _, op := range []string{"chmod", "chown", "utimes", "setxattr", "removexattr"} {
			t.Run(contention+"/"+op, func(t *testing.T) {
				wfs := newInvalidateTestWFS(t)
				wfs.atimeMap = make(map[uint64]time.Time)
				wfs.option.ConcurrentWriters = 1
				wfs.concurrentWriters = util.NewLimitedConcurrentExecutor(1)
				current := attributeAppendEntry(1)
				fake := &attributeCASFiler{entry: proto.Clone(current).(*filer_pb.Entry)}
				entered, resume := make(chan struct{}), make(chan struct{})
				var once sync.Once
				t.Cleanup(func() { once.Do(func() { close(resume) }) })
				fake.afterCommit = func() { close(entered); <-resume }

				startFakeFiler(t, wfs, fake)
				path := util.FullPath("/dir/file")
				wfs.inodeToPath.Lookup(path, 1, false, false, 42, true)
				local := proto.Clone(current).(*filer_pb.Entry)
				wfs.mapPbIdFromFilerToLocal(local)
				fh := newFileHandle(wfs, 1, 42, local)
				fh.RememberPath(path)
				wfs.fhMap.inode2fh[42], wfs.fhMap.fh2inode[1] = fh, 42
				save := &flakySave{}
				fh.dirtyPages.randomWriter.(*ChunkedDirtyPages).saveDataAsChunk = save.saveDataAsChunk
				t.Cleanup(fh.dirtyPages.Destroy)
				done := make(chan fuse.Status, 1)
				go func() {
					if op == "setxattr" {
						done <- wfs.SetXAttr(nil, &fuse.SetXAttrIn{InHeader: fuse.InHeader{NodeId: 42}}, "user.test", []byte("acknowledged"))
					} else if op == "removexattr" {
						done <- wfs.RemoveXAttr(nil, &fuse.InHeader{NodeId: 42}, "user.remove")
					} else {
						in := &fuse.SetAttrIn{}
						in.NodeId = 42
						switch op {
						case "chmod":
							in.Valid, in.Mode = fuse.FATTR_MODE, 0600
						case "chown":
							in.Valid, in.Owner.Uid, in.Owner.Gid = fuse.FATTR_UID|fuse.FATTR_GID, 1001, 21
						case "utimes":
							in.Valid, in.Mtime, in.Mtimensec = fuse.FATTR_MTIME, 123, 456
						}
						done <- wfs.SetAttr(nil, in, &fuse.AttrOut{})
					}
				}()
				select {
				case <-entered:
				case <-time.After(10 * time.Second):
					t.Fatal("attribute RPC never arrived")
				}
				payload := []byte("background write while attribute RPC waits")
				in := &fuse.WriteIn{}
				in.NodeId, in.Fh, in.Offset, in.Size = 42, 1, 256, uint32(len(payload))
				if n, code := wfs.Write(nil, in, payload); code != fuse.OK || n != uint32(len(payload)) {
					t.Fatalf("WRITE: %d %v", n, code)
				}
				flushDone := make(chan fuse.Status, 1)
				flushSettled, publicationEscaped := false, false
				flush := func() { flushDone <- wfs.doFlush(context.Background(), fh, 1000, 20, false, 0) }
				// Probe ownership while the committed response is held. On the old caller
				// no publication boundary exists, so force its whole-entry flush to commit
				// BEFORE releasing the response. With coordination, queue that same flush
				// behind the attribute and force contention at installation/retention.
				table := wfs.metadataPublicationTable()
				probe, unguarded := table.TryAcquireLock("test publication witness", path, util.ExclusiveLock)
				if unguarded {
					table.ReleaseLock(path, probe)
					flush()
					if code := <-flushDone; code != fuse.OK {
						t.Fatalf("pre-response flush: %v", code)
					}
					once.Do(func() { close(resume) })
				} else {
					release := func() {}
					if contention == "installation" {
						active := wfs.fhLockTable.AcquireLock("test installation contention", fh.fh, util.ExclusiveLock)
						release = func() { wfs.fhLockTable.ReleaseLock(fh.fh, active) }
					} else if contention == "retention" {
						fh.entryLock.Lock()
						release = fh.entryLock.Unlock
					}
					go flush()
					// Do not open the RPC response gate merely because flush was started:
					// witness this caller actually asleep on the publication condition.
					if blocked, code := waitFlushPublication(t, flushDone); !blocked {
						flushSettled, publicationEscaped = true, true
						if code != fuse.OK {
							t.Errorf("flush before response gate: %v", code)
						}
					}
					once.Do(func() { close(resume) })
					if contention != "response" {
						frame := "installAckedEntry"
						if contention == "retention" {
							frame = "retainAcknowledgedAttributes"
						}
						// A stack witness proves the real caller reached the held lock. Time
						// only bounds a broken test; no latency threshold decides correctness.
						blocked := waitAttributeFrame(frame)
						if blocked && !flushSettled {
							// Check again before releasing installation/retention. A prematurely
							// unlocked publication lets the real flush finish while this gate is
							// held: join that flush BEFORE retention, forcing the lost-attr order.
							if waiting, code := waitFlushPublication(t, flushDone); !waiting {
								flushSettled, publicationEscaped = true, true
								if code != fuse.OK {
									t.Errorf("flush before %s gate: %v", contention, code)
								}
							}
						}
						release()
						if !blocked {
							t.Fatalf("caller never contended at %s", frame)
						}
					}
				}
				select {
				case code := <-done:
					if code != fuse.OK {
						t.Fatalf("attribute: %v", code)
					}
				case <-time.After(10 * time.Second):
					t.Fatal("attribute did not finish")
				}
				if !unguarded && !flushSettled {
					select {
					case code := <-flushDone:
						if code != fuse.OK {
							t.Fatalf("flush: %v", code)
						}
					case <-time.After(10 * time.Second):
						t.Fatal("flush did not finish")
					}
				}

				fake.mu.Lock()
				defer fake.mu.Unlock()
				switch op {
				case "chmod":
					if fake.entry.Attributes.FileMode != 0600 {
						t.Fatal("flush undid acknowledged chmod")
					}
				case "chown":
					if fake.entry.Attributes.Uid != 1001 || fake.entry.Attributes.Gid != 21 {
						t.Fatal("flush undid acknowledged chown")
					}
				case "utimes":
					if fake.entry.Attributes.Mtime != 123 || fake.entry.Attributes.MtimeNs != 456 {
						t.Fatal("flush undid acknowledged utimes")
					}
				case "setxattr":
					if string(fake.entry.Extended["xattr-user.test"]) != "acknowledged" {
						t.Fatal("flush undid acknowledged setxattr")
					}
				case "removexattr":
					if _, exists := fake.entry.Extended["xattr-user.remove"]; exists {
						t.Fatal("flush undid acknowledged removexattr")
					}
				}
				if len(save.bodies) != 1 || save.bodies[0] != string(payload) {
					t.Fatalf("pending bytes changed: %q", save.bodies)
				}
				if fake.entry.Attributes.FileSize != 256+uint64(len(payload)) || len(fake.entry.Chunks) != 2 {
					t.Fatalf("pending content lost: %v", fake.entry)
				}
				original, appended := false, false
				for _, chunk := range fake.entry.Chunks {
					switch chunk.Offset {
					case 0:
						original = chunk.GetFileIdString() == current.Chunks[0].GetFileIdString() && chunk.Size == 256
					case 256:
						appended = chunk.GetFileIdString() == "1,01" && chunk.Size == uint64(len(payload))
					}
				}
				if !original || !appended {
					t.Fatalf("chunk identity/range changed: %v", fake.entry.Chunks)
				}
				if publicationEscaped {
					t.Fatal("flush escaped publication before its attribute gate opened")
				}

			})
		}
	}
}

func waitAttributeFrame(frame string) bool {
	deadline := time.Now().Add(10 * time.Second)
	for time.Now().Before(deadline) {
		buf := make([]byte, 1<<20)
		n := runtime.Stack(buf, true)
		if strings.Contains(string(buf[:n]), "."+frame+"(") {
			return true
		}
		runtime.Gosched()
	}
	return false
}

// Require the flush goroutine's waiting state AND both publication frames.
// Seeing a frame alone also admits a runnable waiter awakened by an early
// unlock. Broadcast makes that goroutine runnable before retention is reached,
// so the early-unlock mutant must progress to its real CreateEntry instead.
func waitFlushPublication(t *testing.T, done <-chan fuse.Status) (bool, fuse.Status) {
	t.Helper()
	deadline := time.Now().Add(10 * time.Second)
	for time.Now().Before(deadline) {
		select {
		case code := <-done:
			return false, code
		default:
		}
		buf := make([]byte, 1<<20)
		n := runtime.Stack(buf, true)
		for _, stack := range strings.Split(string(buf[:n]), "\n\n") {
			header, _, _ := strings.Cut(stack, "\n")
			if strings.Contains(header, "[sync.Cond.Wait]") &&
				strings.Contains(stack, ".flushMetadataToFiler(") && strings.Contains(stack, ".lockMetadataPublication(") {
				t.Log("flush arrived and is blocked at publication before opening gate")
				return true, fuse.OK
			}
		}
		runtime.Gosched()
	}
	t.Error("flush neither blocked at publication nor settled")
	return false, fuse.EIO
}

// The filer applies the update even though the caller's syscall was
// interrupted while the RPC was in flight.
type interruptingAttributeFiler struct {
	*attributeCASFiler
	once      sync.Once
	interrupt chan struct{}
}

func (s *interruptingAttributeFiler) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
	s.once.Do(func() { close(s.interrupt) })
	select {
	case <-ctx.Done():
	case <-time.After(100 * time.Millisecond):
	}
	return s.attributeCASFiler.UpdateEntry(context.Background(), req)
}

func TestAttributeSaveSurvivesInterrupt(t *testing.T) {
	for _, op := range []string{"chmod", "setxattr"} {
		t.Run(op, func(t *testing.T) {
			wfs := newInvalidateTestWFS(t)
			wfs.atimeMap = make(map[uint64]time.Time)
			current := attributeAppendEntry(300)
			fake := &interruptingAttributeFiler{attributeCASFiler: &attributeCASFiler{entry: proto.Clone(current).(*filer_pb.Entry)}, interrupt: make(chan struct{})}
			startFakeFiler(t, wfs, fake)
			path := util.FullPath("/dir/file")
			wfs.inodeToPath.Lookup(util.FullPath("/dir"), 1, true, false, 0, false)
			wfs.inodeToPath.Lookup(path, 1, false, false, 42, true)
			wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir"))
			if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry("/dir", current), 1); err != nil {
				t.Fatal(err)
			}
			var code fuse.Status
			if op == "setxattr" {
				in := &fuse.SetXAttrIn{}
				in.NodeId = 42
				code = wfs.SetXAttr(fake.interrupt, in, "user.test", []byte("new"))
			} else {
				in := &fuse.SetAttrIn{}
				in.NodeId = 42
				in.Valid, in.Mode = fuse.FATTR_MODE, 0600
				code = wfs.SetAttr(fake.interrupt, in, &fuse.AttrOut{})
			}
			if code != fuse.OK {
				t.Fatalf("interrupted %s reported %v for an update the filer applied", op, code)
			}
			cached, _, err := wfs.metaCache.FindEntry(context.Background(), path)
			if err != nil {
				t.Fatal(err)
			}
			if op == "chmod" && cached.Attr.Mode.Perm() != 0600 {
				t.Fatalf("local cache mode %v, want 0600", cached.Attr.Mode.Perm())
			}
			if op == "setxattr" && string(cached.Extended["xattr-user.test"]) != "new" {
				t.Fatal("local cache missing the saved xattr")
			}
		})
	}
}

// The test mount maps local uid 1000 to filer uid 2000, so a file the filer
// records as uid 1000 reads back as local 1000 and maps forward to 2000.
func TestAttributeSaveKeepsCollidingFilerIds(t *testing.T) {
	for _, op := range []string{"chmod", "setxattr"} {
		t.Run(op, func(t *testing.T) {
			wfs := newInvalidateTestWFS(t)
			wfs.atimeMap = make(map[uint64]time.Time)
			current := attributeAppendEntry(3)
			current.Attributes.Uid = 1000
			fake := &attributeCASFiler{entry: proto.Clone(current).(*filer_pb.Entry)}
			startFakeFiler(t, wfs, fake)
			path := util.FullPath("/dir/file")
			wfs.inodeToPath.Lookup(util.FullPath("/dir"), 1, true, false, 0, false)
			wfs.inodeToPath.Lookup(path, 1, false, false, 42, true)
			wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir"))
			if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry("/dir", current), 1); err != nil {
				t.Fatal(err)
			}
			var code fuse.Status
			if op == "setxattr" {
				in := &fuse.SetXAttrIn{}
				in.NodeId = 42
				code = wfs.SetXAttr(nil, in, "user.test", []byte("new"))
			} else {
				in := &fuse.SetAttrIn{}
				in.NodeId = 42
				in.Valid, in.Mode = fuse.FATTR_MODE, 0600
				code = wfs.SetAttr(nil, in, &fuse.AttrOut{})
			}
			if code != fuse.OK {
				t.Fatalf("%s on a colliding uid: %v", op, code)
			}
			fake.mu.Lock()
			defer fake.mu.Unlock()
			if fake.entry.Attributes.Uid != 1000 {
				t.Fatalf("%s rewrote the filer uid to %d", op, fake.entry.Attributes.Uid)
			}
			cached, _, err := wfs.metaCache.FindEntry(context.Background(), path)
			if err != nil {
				t.Fatal(err)
			}
			if cached.Attr.Uid != 1000 {
				t.Fatalf("local cache uid %d, want 1000", cached.Attr.Uid)
			}
		})
	}
}

// The seed filer has not applied the owner's latest update yet and forwards
// conditional writes to the owner, as filers do under the lock ring.
type laggingSeedFiler struct {
	filer_pb.UnimplementedSeaweedFilerServer
	stale *filer_pb.Entry
	owner *attributeCASFiler
}

func (s *laggingSeedFiler) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
	return &filer_pb.LookupDirectoryEntryResponse{Entry: proto.Clone(s.stale).(*filer_pb.Entry)}, nil
}

func (s *laggingSeedFiler) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
	return s.owner.UpdateEntry(ctx, req)
}

func TestAttributeRetryReadsOwnerFiler(t *testing.T) {
	wfs := newInvalidateTestWFS(t)
	wfs.atimeMap = make(map[uint64]time.Time)
	stale := attributeAppendEntry(1)
	owner := &attributeCASFiler{entry: attributeAppendEntry(300)}
	startFakeFiler(t, wfs, owner)
	ownerAddress := wfs.option.FilerAddresses[0]
	startFakeFiler(t, wfs, &laggingSeedFiler{stale: stale, owner: owner})
	wfs.lockClient = cluster.NewLockClient(nil, wfs.option.FilerAddresses[0])
	wfs.lockClient.SetRing([]pb.ServerAddress{ownerAddress}, 1)

	path := util.FullPath("/dir/file")
	wfs.inodeToPath.Lookup(util.FullPath("/dir"), 1, true, false, 0, false)
	wfs.inodeToPath.Lookup(path, 1, false, false, 42, true)
	wfs.inodeToPath.MarkChildrenCached(util.FullPath("/dir"))
	if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry("/dir", stale), 1); err != nil {
		t.Fatal(err)
	}
	in := &fuse.SetAttrIn{}
	in.NodeId = 42
	in.Valid, in.Mode = fuse.FATTR_MODE, 0600
	if code := wfs.SetAttr(nil, in, &fuse.AttrOut{}); code != fuse.OK {
		t.Fatalf("chmod behind a lagging seed filer: %v", code)
	}
	owner.mu.Lock()
	defer owner.mu.Unlock()
	if len(owner.entry.Chunks) != 300 || owner.entry.Attributes.FileMode&0777 != 0600 {
		t.Fatalf("owner entry: %d chunks, mode %o", len(owner.entry.Chunks), owner.entry.Attributes.FileMode)
	}
}