mount,filer: re-assert POSIX locks via keepalive (ownership migration + restart) (#9668)

* mount: renew POSIX lock leases via keepalive

The mount tracks the inode keys it holds locks on and a background loop
renews its session lease (KEEP_ALIVE) with each key's owner filer every
5s, within the filer's 15s TTL. A live mount is never reaped; a dead one
stops renewing and owners reclaim its locks. Tracking is a superset:
holds are added on grant and dropped only on owner release, so a still
held lock is never under-renewed.

* mount,filer: re-assert held POSIX locks via keepalive

The owner filer holds POSIX advisory locks as in-memory soft state, so a key's
owner change (ring rebalance) or an owner restart lost or stranded them: the new
or restarted owner was blind to existing holders and would double-grant.

Make the keepalive carry the mount's held lock ranges per key. The mount mirrors
its own granted locks (posixOwn), and each tick re-asserts them to the key's
current owner, which rebuilds that session's locks from the assertion — self
-healing after a takeover or restart. The owner arbitrates re-asserted locks
against other sessions so it never double-grants; a lock that lost a migration
race is reported, not forced. A bare keepalive (no ranges) still just renews.

weed/server/filer_grpc_server_posix_lock.go

@@ -61,6 +61,7 @@ func (fs *FilerServer) PosixLock(ctx context.Context, req *filer_pb.PosixLockReq
 				IsMoved: true,
 				Op:      req.Op,
 				Lock:    req.Lock,
+				Locks:   req.Locks,
 			}
 			glog.V(4).InfofCtx(ctx, "PosixLock %s op=%v: forwarding to owner %s", req.Key, req.Op, owner)
 			var resp *filer_pb.PosixLockResponse
@@ -107,13 +108,38 @@ func (fs *FilerServer) PosixLock(ctx context.Context, req *filer_pb.PosixLockReq
 	case filer_pb.PosixLockOp_RELEASE_FLOCK_OWNER:
 		fs.posixLocks.ReleaseFlockOwner(req.Key, lk.Sid, lk.Owner)
 	case filer_pb.PosixLockOp_KEEP_ALIVE:
-		fs.posixLocks.Renew(lk.Sid)
+		// A re-assertion carries the mount's held locks on this key so the owner
+		// can rebuild its in-memory state after an ownership change or restart; a
+		// bare keepalive just renews the lease.
+		if len(req.Locks) > 0 {
+			held := make([]posixlock.Range, 0, len(req.Locks))
+			for _, l := range req.Locks {
+				held = append(held, posixRangeFromPb(l))
+			}
+			if conflicts := fs.posixLocks.Reassert(req.Key, lk.Sid, held); len(conflicts) > 0 {
+				glog.Warningf("posix reassert %s sid %d: %d lock(s) lost to another session: %+v", req.Key, lk.Sid, len(conflicts), conflicts)
+			}
+		} else {
+			fs.posixLocks.Renew(lk.Sid)
+		}
 	default:
 		return &filer_pb.PosixLockResponse{}, fmt.Errorf("unknown posix lock op %v", req.Op)
 	}
 	return resp, nil
 }
 
+func posixRangeFromPb(l *filer_pb.PosixLockRange) posixlock.Range {
+	return posixlock.Range{
+		Start:   l.GetStart(),
+		End:     l.GetEnd(),
+		Type:    l.GetType(),
+		Sid:     l.GetSid(),
+		Owner:   l.GetOwner(),
+		Pid:     l.GetPid(),
+		IsFlock: l.GetIsFlock(),
+	}
+}
+
 func posixRangeToPb(r posixlock.Range) *filer_pb.PosixLockRange {
 	return &filer_pb.PosixLockRange{
 		Start:   r.Start,