seaweedfs/weed/mount/weedfs_posix_lock_routed.go

package mount

import (
	"context"
	"encoding/binary"
	"encoding/hex"
	"math/rand/v2"
	"sync"
	"time"

	"github.com/seaweedfs/go-fuse/v2/fuse"
	"github.com/seaweedfs/seaweedfs/weed/filer/posixlock"
	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

// randomPosixSid returns a process-stable, cluster-unique-enough session id for
// this mount, used to namespace lock owners so the same FUSE owner value on a
// different mount never aliases, and to scope lease-based reaping.
func randomPosixSid() uint64 {
	return binary.BigEndian.Uint64(util.RandomBytes(8))
}

// Routed POSIX locking: when -dlm is set, advisory locks are serialized on the
// inode's owner filer instead of in this mount's local table, so locks are
// honored across mounts. The mount calls its filer and relies on filer-side
// forwarding to reach the owner. Blocking (SetLkw) is client-side polling — there
// is no server-side wait queue.

const (
	posixLockMinBackoff = 5 * time.Millisecond
	posixLockMaxBackoff = 200 * time.Millisecond
	posixLockKeyPrefix  = "s3.fuse.lock:"
	// posixLockReleaseTimeout bounds the background unlock/release RPCs. They run
	// off the syscall path (close/flush) and must not be cancelled by an
	// interrupt, but a deadline keeps a slow or unreachable filer from blocking
	// close indefinitely. It also bounds each keepalive RPC.
	posixLockReleaseTimeout = 5 * time.Second
	// posixKeepaliveInterval renews the session lease well within the filer's
	// posixLockSessionTTL (15s) so a live mount is never reaped.
	posixKeepaliveInterval = 5 * time.Second
)

// posixLockHint records, per inode, which owners this mount has taken fcntl locks
// for. Flush consults it to force a synchronous close and route a release without
// an RPC on every close(). It is a superset hint: an extra sync flush or a no-op
// release RPC is harmless, a missed release is not — so it is added on grant and
// cleared on close (which drops the owner's POSIX locks per POSIX semantics).
type posixLockHint struct {
	mu sync.Mutex
	m  map[uint64]map[uint64]struct{}
}

func newPosixLockHint() *posixLockHint {
	return &posixLockHint{m: make(map[uint64]map[uint64]struct{})}
}

func (h *posixLockHint) add(inode, owner uint64) {
	h.mu.Lock()
	defer h.mu.Unlock()
	owners := h.m[inode]
	if owners == nil {
		owners = make(map[uint64]struct{})
		h.m[inode] = owners
	}
	owners[owner] = struct{}{}
}

func (h *posixLockHint) has(inode, owner uint64) bool {
	h.mu.Lock()
	defer h.mu.Unlock()
	_, ok := h.m[inode][owner]
	return ok
}

func (h *posixLockHint) drop(inode, owner uint64) {
	h.mu.Lock()
	defer h.mu.Unlock()
	if owners := h.m[inode]; owners != nil {
		delete(owners, owner)
		if len(owners) == 0 {
			delete(h.m, inode)
		}
	}
}

// posixLockKeyForInode resolves a FUSE inode to its cluster-stable lock identity:
// the HardLinkId for a hardlinked file (so all names share one owner and table),
// else the path. POSIX locks are inode-scoped, so this must never be the FUSE
// NodeId (mount-local) for a multi-named inode.
func (wfs *WFS) posixLockKeyForInode(inode uint64) (string, bool) {
	path, status := wfs.inodeToPath.GetPath(inode)
	if status != fuse.OK {
		return "", false
	}
	if entry, _, st := wfs.maybeLoadEntry(context.Background(), path); st == fuse.OK && entry != nil && len(entry.HardLinkId) > 0 {
		return posixLockKeyPrefix + "hl:" + hex.EncodeToString(entry.HardLinkId), true
	}
	return posixLockKeyPrefix + string(path), true
}

func posixLockTypeToWire(typ uint32) uint32 {
	switch typ {
	case f_RDLCK:
		return posixlock.Read
	case f_WRLCK:
		return posixlock.Write
	default:
		return posixlock.Unlock
	}
}

func posixLockTypeFromWire(typ uint32) uint32 {
	switch typ {
	case posixlock.Read:
		return f_RDLCK
	case posixlock.Write:
		return f_WRLCK
	default:
		return f_UNLCK
	}
}

func (wfs *WFS) posixRangeFromLkIn(in *fuse.LkIn) posixlock.Range {
	return posixlock.Range{
		Start:   in.Lk.Start,
		End:     in.Lk.End,
		Type:    posixLockTypeToWire(in.Lk.Typ),
		Sid:     wfs.posixSid,
		Owner:   in.Owner,
		Pid:     in.Lk.Pid,
		IsFlock: in.LkFlags&fuse.FUSE_LK_FLOCK != 0,
	}
}

// posixLockContext derives a context from the FUSE cancel channel so an
// interrupted lock syscall aborts its in-flight RPC. The caller must call the
// returned func to release the watcher goroutine.
func posixLockContext(cancel <-chan struct{}) (context.Context, context.CancelFunc) {
	ctx, cancelFn := context.WithCancel(context.Background())
	if cancel != nil {
		go func() {
			select {
			case <-cancel:
				cancelFn()
			case <-ctx.Done():
			}
		}()
	}
	return ctx, cancelFn
}

func (wfs *WFS) callPosixLock(ctx context.Context, key string, op filer_pb.PosixLockOp, lk posixlock.Range) (*filer_pb.PosixLockResponse, error) {
	var resp *filer_pb.PosixLockResponse
	err := wfs.withFilerClient(ctx, false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
		var e error
		resp, e = client.PosixLock(ctx, &filer_pb.PosixLockRequest{
			Key: key,
			Op:  op,
			Lock: &filer_pb.PosixLockRange{
				Start: lk.Start, End: lk.End, Type: lk.Type,
				Sid: lk.Sid, Owner: lk.Owner, Pid: lk.Pid, IsFlock: lk.IsFlock,
			},
		})
		return e
	})
	return resp, err
}

func (wfs *WFS) routedGetLk(cancel <-chan struct{}, in *fuse.LkIn, out *fuse.LkOut) fuse.Status {
	key, ok := wfs.posixLockKeyForInode(in.NodeId)
	if !ok {
		return fuse.EINVAL
	}
	ctx, done := posixLockContext(cancel)
	defer done()
	resp, err := wfs.callPosixLock(ctx, key, filer_pb.PosixLockOp_GET_LK, wfs.posixRangeFromLkIn(in))
	if err != nil {
		glog.Warningf("routed GetLk %s: %v", key, err)
		return fuse.EIO
	}
	if resp.GetHasConflict() {
		c := resp.GetConflict()
		out.Lk.Start, out.Lk.End, out.Lk.Pid = c.GetStart(), c.GetEnd(), c.GetPid()
		out.Lk.Typ = posixLockTypeFromWire(c.GetType())
	} else {
		out.Lk.Typ = f_UNLCK
	}
	return fuse.OK
}

func (wfs *WFS) routedSetLk(cancel <-chan struct{}, in *fuse.LkIn) fuse.Status {
	key, ok := wfs.posixLockKeyForInode(in.NodeId)
	if !ok {
		return fuse.EINVAL
	}
	lk := wfs.posixRangeFromLkIn(in)
	if lk.Type == posixlock.Unlock {
		return wfs.routedUnlock(key, lk)
	}
	ctx, done := posixLockContext(cancel)
	defer done()
	resp, err := wfs.callPosixLock(ctx, key, filer_pb.PosixLockOp_TRY_LOCK, lk)
	if err != nil {
		glog.Warningf("routed SetLk %s: %v", key, err)
		return fuse.EIO
	}
	if !resp.GetGranted() {
		return fuse.EAGAIN
	}
	wfs.recordPosixGrant(key, in.NodeId, lk)
	return fuse.OK
}

func (wfs *WFS) routedSetLkw(cancel <-chan struct{}, in *fuse.LkIn) fuse.Status {
	key, ok := wfs.posixLockKeyForInode(in.NodeId)
	if !ok {
		return fuse.EINVAL
	}
	lk := wfs.posixRangeFromLkIn(in)
	if lk.Type == posixlock.Unlock {
		return wfs.routedUnlock(key, lk)
	}
	ctx, done := posixLockContext(cancel)
	defer done()
	status := posixPollAcquire(cancel, func() (bool, error) {
		resp, err := wfs.callPosixLock(ctx, key, filer_pb.PosixLockOp_TRY_LOCK, lk)
		if err != nil {
			return false, err
		}
		return resp.GetGranted(), nil
	})
	if status == fuse.OK {
		wfs.recordPosixGrant(key, in.NodeId, lk)
	}
	return status
}

// recordPosixGrant notes a newly granted lock: the flush hint (fcntl only) and
// the own-lock mirror, which the keepalive re-asserts to the key's owner so the
// lease is renewed and the owner can rebuild after a takeover or restart.
func (wfs *WFS) recordPosixGrant(key string, inode uint64, lk posixlock.Range) {
	if !lk.IsFlock {
		wfs.posixHint.add(inode, lk.Owner)
	}
	wfs.posixOwn.Track(key, lk)
}

func (wfs *WFS) routedUnlock(key string, lk posixlock.Range) fuse.Status {
	// Drop the range from the mirror first so a keepalive re-assertion can't
	// resurrect it; if the RPC below fails, the next re-assertion reconciles the
	// owner to this (released) state anyway.
	wfs.posixOwn.Unlock(key, lk)
	// A release must complete even if the syscall was interrupted; cancelling it
	// would leak the lock on the owner filer. Bound it so a stuck filer can't
	// hang close() forever.
	ctx, cancel := context.WithTimeout(context.Background(), posixLockReleaseTimeout)
	defer cancel()
	if _, err := wfs.callPosixLock(ctx, key, filer_pb.PosixLockOp_UNLOCK, lk); err != nil {
		glog.Warningf("routed unlock %s: %v", key, err)
		return fuse.EIO
	}
	return fuse.OK
}

// posixPollAcquire retries try with capped, jittered backoff until it is granted,
// the cancel channel fires (EINTR), or try errors (EIO). This is the client-side
// stand-in for a server-side wait queue.
func posixPollAcquire(cancel <-chan struct{}, try func() (bool, error)) fuse.Status {
	backoff := posixLockMinBackoff
	for {
		select {
		case <-cancel:
			return fuse.EINTR
		default:
		}
		granted, err := try()
		if err != nil {
			return fuse.EIO
		}
		if granted {
			return fuse.OK
		}
		timer := time.NewTimer(backoff + time.Duration(rand.Int64N(int64(backoff))))
		select {
		case <-cancel:
			timer.Stop()
			return fuse.EINTR
		case <-timer.C:
		}
		if backoff < posixLockMaxBackoff {
			if backoff *= 2; backoff > posixLockMaxBackoff {
				backoff = posixLockMaxBackoff
			}
		}
	}
}

func (wfs *WFS) routedReleasePosixOwner(inode, owner uint64) bool {
	if !wfs.posixHint.has(inode, owner) {
		return true
	}
	key, ok := wfs.posixLockKeyForInode(inode)
	if !ok {
		return false
	}
	wfs.posixOwn.ReleasePosixOwner(key, wfs.posixSid, owner)
	ctx, cancel := context.WithTimeout(context.Background(), posixLockReleaseTimeout)
	defer cancel()
	if _, err := wfs.callPosixLock(ctx, key, filer_pb.PosixLockOp_RELEASE_POSIX_OWNER, posixlock.Range{Sid: wfs.posixSid, Owner: owner}); err != nil {
		// Keep the hint so a later flush retries the release; dropping it on a
		// transient failure would strand the lock until the owner filer's
		// session-lease reaping expires it.
		glog.Warningf("routed release posix owner %s: %v", key, err)
		return false
	}
	wfs.posixHint.drop(inode, owner)
	return true
}

func (wfs *WFS) routedReleaseFlockOwner(inode, owner uint64) {
	key, ok := wfs.posixLockKeyForInode(inode)
	if !ok {
		return
	}
	wfs.posixOwn.ReleaseFlockOwner(key, wfs.posixSid, owner)
	ctx, cancel := context.WithTimeout(context.Background(), posixLockReleaseTimeout)
	defer cancel()
	if _, err := wfs.callPosixLock(ctx, key, filer_pb.PosixLockOp_RELEASE_FLOCK_OWNER, posixlock.Range{Sid: wfs.posixSid, Owner: owner}); err != nil {
		glog.Warningf("routed release flock owner %s: %v", key, err)
	}
}

// crossMountLocks reports whether advisory locks are routed to the inode's owner
// filer rather than served from the per-mount local table. It tracks -dlm: lock
// coordination rides the same switch as whole-file write coordination, and is
// off under writeback cache (which implies single-writer, so lockClient is nil).
func (wfs *WFS) crossMountLocks() bool {
	return wfs.lockClient != nil
}

// releasePosixOwner / releaseFlockOwner / hasPosixOwner dispatch to the routed
// authority or the local table based on crossMountLocks, keeping the Flush and
// Release call sites flag-agnostic.

// releasePosixOwner reports whether the owner's locks were released. A false
// return means a routed release failed and the close-time owner records must
// be kept so a later Flush or the final Release retries instead of stranding
// a live lock on the owner filer.
func (wfs *WFS) releasePosixOwner(inode, owner uint64) bool {
	if wfs.crossMountLocks() {
		return wfs.routedReleasePosixOwner(inode, owner)
	}
	wfs.posixLocks.ReleasePosixOwner(inode, owner)
	return true
}

func (wfs *WFS) releaseFlockOwner(inode, owner uint64) {
	if wfs.crossMountLocks() {
		wfs.routedReleaseFlockOwner(inode, owner)
		return
	}
	wfs.posixLocks.ReleaseFlockOwner(inode, owner)
}

func (wfs *WFS) hasPosixOwner(inode, owner uint64) bool {
	if wfs.crossMountLocks() {
		return owner != 0 && wfs.posixHint.has(inode, owner)
	}
	return wfs.posixLocks.HasPosixOwner(inode, owner)
}

// callPosixReassert sends the mount's held locks on key to the key's current
// owner filer as a KEEP_ALIVE, which renews the lease and lets the owner rebuild
// its in-memory state after a takeover or restart.
func (wfs *WFS) callPosixReassert(ctx context.Context, key string, locks []posixlock.Range) error {
	pbLocks := make([]*filer_pb.PosixLockRange, 0, len(locks))
	for _, l := range locks {
		pbLocks = append(pbLocks, &filer_pb.PosixLockRange{
			Start: l.Start, End: l.End, Type: l.Type,
			Sid: l.Sid, Owner: l.Owner, Pid: l.Pid, IsFlock: l.IsFlock,
		})
	}
	return wfs.withFilerClient(ctx, false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
		_, e := client.PosixLock(ctx, &filer_pb.PosixLockRequest{
			Key:   key,
			Op:    filer_pb.PosixLockOp_KEEP_ALIVE,
			Lock:  &filer_pb.PosixLockRange{Sid: wfs.posixSid},
			Locks: pbLocks,
		})
		return e
	})
}

// posixKeepaliveConcurrency bounds the parallel keepalive RPCs per tick. A mount
// holding locks on many inodes must renew every lease well within the filer TTL;
// dispatching them concurrently keeps the round trip from scaling with lock count.
const posixKeepaliveConcurrency = 32

// loopRenewPosixLeases re-asserts this mount's held locks to the owner filer of
// every key, which renews the session lease and rebuilds the owner's state after
// a ring change or owner restart. A dead mount stops re-asserting and the owners'
// sweepers reclaim its locks after the TTL.
func (wfs *WFS) loopRenewPosixLeases() {
	ticker := time.NewTicker(posixKeepaliveInterval)
	defer ticker.Stop()
	for range ticker.C {
		held := wfs.posixOwn.Snapshot()
		var wg sync.WaitGroup
		sem := make(chan struct{}, posixKeepaliveConcurrency)
		for key, locks := range held {
			wg.Add(1)
			sem <- struct{}{}
			go func() {
				defer wg.Done()
				defer func() { <-sem }()
				// Bound each re-assertion so a stuck filer can't block wg.Wait and
				// stall the whole tick, which would let other keys' leases expire
				// and get reaped.
				ctx, cancel := context.WithTimeout(context.Background(), posixLockReleaseTimeout)
				defer cancel()
				if err := wfs.callPosixReassert(ctx, key, locks); err != nil {
					glog.V(2).Infof("posix reassert %s: %v", key, err)
				}
			}()
		}
		wg.Wait()
	}
}