Group voice and video calls with MatrixRTC and LiveKit

Every call in the web client now goes through MatrixRTC with a LiveKit
SFU, the model Element Call and Element X use; the legacy 1:1 m.call.*
calls are gone. In a room of two the other person is rung and can answer
or decline; in a bigger room a banner offers to join the call. Calls show
a grid of participant tiles with mute, camera, screen sharing and leave,
and media is end-to-end encrypted in encrypted rooms.

Server:
- MSC4140 delayed events (send/state with a delay, plus the
  delayed_events list/restart/cancel/send API), stored and scheduled, so
  a member whose client drops off is taken out of the call.
- POST /user/{userId}/openid/request_token, which lk-jwt-service uses to
  hand out LiveKit tokens.
- org.matrix.msc4143.rtc_foci in /.well-known/matrix/client, configured
  under client_api.rtc.
- Room presets let members send call membership state.
- Migration 0014 for both SQL dialects.

Deploy: LiveKit and lk-jwt-service in compose (calls.yml) and the helm
chart. Docs: docs/calls.md, configuration and deployment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

apps/web/src/stores/call.ts

@@ -1,7 +1,26 @@
-import { CallEvent, CallFeedEvent, createNewMatrixCall, type MatrixCall, type MatrixClient } from 'matrix-js-sdk'
-import { CallDirection, CallErrorCode, CallParty, CallState, CallType, type CallError } from 'matrix-js-sdk/lib/webrtc/call'
-import { CallEventHandlerEvent } from 'matrix-js-sdk/lib/webrtc/callEventHandler'
-import type { CallFeed } from 'matrix-js-sdk/lib/webrtc/callFeed'
+import {
+  ClientEvent,
+  EventType,
+  MatrixEventEvent,
+  RelationType,
+  RoomEvent as MatrixRoomEvent,
+  SyncState,
+  type IRoomTimelineData,
+  type MatrixClient,
+  type MatrixEvent,
+  type Room as MatrixRoom,
+} from 'matrix-js-sdk'
+import { MatrixRTCSessionEvent, MatrixRTCSessionManagerEvents, type MatrixRTCSession } from 'matrix-js-sdk/lib/matrixrtc'
+import {
+  BaseKeyProvider,
+  createKeyMaterialFromBuffer,
+  DisconnectReason,
+  isE2EESupported,
+  Room,
+  RoomEvent,
+  Track,
+  type Participant,
+} from 'livekit-client'
 import { defineStore } from 'pinia'
 import { computed, ref, shallowRef } from 'vue'
 import { errorMessage } from '@/lib/format'
@@ -9,332 +28,665 @@ import { client } from '@/matrix/client'
 import { startTone, stopTone } from '@/matrix/tones'
 import { useToasts } from '@/stores/toasts'
 
-// Voice and video calls between two people: Matrix's 1:1 VoIP (m.call.*
-// events in the room, WebRTC between the browsers), which the SDK runs.
-// In an encrypted room the call events are encrypted like any other; the
-// media itself is always encrypted by WebRTC (DTLS-SRTP).
+// Voice and video calls, all of them: MatrixRTC (MSC4143) with a LiveKit
+// SFU, the way Element Call does them. A call between two people is the
+// same thing as a call between ten, with a ring on top.
 //
-// One call at a time. A call that comes in while one is going is answered
-// "busy", as a phone would.
+// Who is in a room's call is Matrix state (one
+// org.matrix.msc3401.call.member event per device), which the SDK's
+// MatrixRTC session keeps; the media goes through LiveKit, whose token the
+// LiveKit JWT service hands out for an OpenID token from the homeserver.
+// The session also schedules a delayed "left" with the homeserver (MSC4140)
+// and keeps pushing it back, so a member whose browser crashes or loses
+// its connection drops out of the call on its own.
+//
+// Starting a call in a room with one other person rings them: the first
+// member sends an org.matrix.msc4075.rtc.notification, and the other side
+// answers by joining or declines with org.matrix.msc4310.rtc.decline.
+//
+// In an encrypted room the media is end-to-end encrypted too: each member
+// makes its own key, shares it with the others over encrypted to-device
+// messages, and LiveKit encrypts every frame with it (encryptMedia).
+//
+// One call at a time.
 
-export type CallPhase = 'ringing' | 'calling' | 'connecting' | 'connected' | 'ended'
+/** A person (well, a device) in the call. */
+export interface Tile {
+  /** The LiveKit identity: "@user:server:DEVICE". */
+  id: string
+  userId: string
+  name: string
+  avatarMxc?: string
+  local: boolean
+  camera?: Track
+  screen?: Track
+  /** A remote microphone, to play. Your own is never played back. */
+  audio?: Track
+  micMuted: boolean
+  speaking: boolean
+}
 
-/** How long the ended call stays on screen, saying why. */
-const endedLingerMs = 2500
+export type CallPhase = 'joining' | 'connected' | 'reconnecting'
+
+/** A call ringing for you. */
+export interface IncomingCall {
+  roomId: string
+  /** The notification event, which a decline refers to. */
+  eventId: string
+  callerId: string
+  callerName: string
+  callerAvatarMxc?: string
+  roomName: string
+  video: boolean
+  /** When the ring stops on its own (ms since the epoch). */
+  expiresAt: number
+}
+
+/** The MatrixRTC transport this client publishes: LiveKit via the JWT service. */
+interface LivekitTransport {
+  type: 'livekit'
+  livekit_service_url: string
+  livekit_alias: string
+  [key: string]: unknown
+}
+
+/** How long a call rings, unless the notification says less. */
+const ringLifetimeMs = 60_000
+/** The most a received notification may ring for, whatever it says. */
+const maxRingMs = 120_000
+
+/**
+ * Feeds the per-participant keys MatrixRTC hands out to LiveKit's frame
+ * encryption. Each key belongs to one participant and index; there is no
+ * shared key and no ratcheting, the same settings as Element Call, so the
+ * two can be in one call.
+ */
+class MatrixKeyProvider extends BaseKeyProvider {
+  constructor() {
+    super({ ratchetWindowSize: 0, keyringSize: 256 })
+  }
+
+  async setParticipantKey(key: Uint8Array, index: number, identity: string) {
+    const material = await createKeyMaterialFromBuffer(key.buffer.slice(key.byteOffset, key.byteOffset + key.byteLength) as ArrayBuffer)
+    this.onSetEncryptionKey(material, identity, index)
+  }
+}
+
+/** The user of a LiveKit identity lk-jwt-service made: "@alice:example.org:DEVICE". */
+function identityUser(identity: string): string {
+  // The device is after the last colon; a server name may itself have a port.
+  const i = identity.lastIndexOf(':')
+  return i > 0 ? identity.slice(0, i) : identity
+}
+
+/** How many people (not devices) are in a session. */
+function countMembers(session: MatrixRTCSession): number {
+  return new Set(session.memberships.map((m) => m.userId)).size
+}
+
+/** The other person in a room of two, or undefined. */
+function otherOfTwo(room: MatrixRoom, me: string): string | undefined {
+  if (room.getJoinedMemberCount() !== 2) return undefined
+  return room.getJoinedMembers().find((m) => m.userId !== me)?.userId
+}
 
 export const useCall = defineStore('call', () => {
-  const call = shallowRef<MatrixCall | null>(null)
+  /** The room whose call this client is in. */
+  const roomId = ref<string | null>(null)
+  const phase = ref<CallPhase | null>(null)
   /** Bumps on every change the call view shows. */
   const version = ref(0)
-  /** Seconds since the call connected. */
+  /** How many people are in each room's call, for every room with one. */
+  const active = ref<Record<string, number>>({})
+  const lk = shallowRef<Room | null>(null)
+  /** Media is end-to-end encrypted. */
+  const encrypted = ref(false)
+  /** Joined with the camera on. */
+  const video = ref(false)
+  /** Seconds since you joined. */
   const length = ref(0)
-  /** Why the call ended, once it has. */
-  const endReason = ref('')
+  /** We rang someone and they haven't come yet. */
+  const ringingOut = ref(false)
+  /** A call ringing for you. */
+  const incoming = ref<IncomingCall | null>(null)
+
   let attached: MatrixClient | null = null
-  let clearTimer = 0
-  /** The call got as far as connecting media, at some point. */
-  let connected = false
-  /** The feeds whose mute state is listened to. */
-  let feeds: CallFeed[] = []
-
-  const phase = computed<CallPhase | null>(() => {
-    void version.value
-    const c = call.value
-    if (!c) return null
-    switch (c.state) {
-      case CallState.Ringing:
-        return 'ringing'
-      case CallState.Fledgling:
-      case CallState.WaitLocalMedia:
-      case CallState.CreateOffer:
-      case CallState.InviteSent:
-        return c.direction === CallDirection.Inbound ? 'connecting' : 'calling'
-      case CallState.CreateAnswer:
-      case CallState.Connecting:
-        return 'connecting'
-      case CallState.Connected:
-        return 'connected'
-      default:
-        return 'ended'
-    }
-  })
-
-  const isVideo = computed(() => {
-    void version.value
-    return call.value?.type === CallType.Video
-  })
-
-  const peerName = computed(() => {
-    void version.value
-    const c = call.value
-    if (!c) return ''
-    const member = c.getOpponentMember()
-    if (member) return member.name
-    const room = attached?.getRoom(c.roomId)
-    return room?.name ?? c.invitee ?? ''
-  })
-
-  const peerAvatarMxc = computed(() => {
-    void version.value
-    return call.value?.getOpponentMember()?.getMxcAvatarUrl() ?? undefined
-  })
+  let session: MatrixRTCSession | null = null
+  let keyProvider: MatrixKeyProvider | null = null
+  let lengthTimer = 0
+  let ringOutTimer = 0
+  let ringInTimer = 0
+  /** The ring notification we sent, which a decline refers to. */
+  let sentNotification: string | null = null
+  const watched = new Map<string, MatrixRTCSession>()
 
   function bump() {
     version.value++
   }
 
-  // A feed muting its camera or microphone is not a change in the call's
-  // feeds, so each feed is listened to as well.
-  function watchFeeds(c: MatrixCall | null) {
-    for (const f of feeds) {
-      f.off(CallFeedEvent.MuteStateChanged, bump)
-      f.off(CallFeedEvent.NewStream, bump)
-    }
-    feeds = c ? c.getFeeds() : []
-    for (const f of feeds) {
-      f.on(CallFeedEvent.MuteStateChanged, bump)
-      f.on(CallFeedEvent.NewStream, bump)
-    }
+  const joined = computed(() => roomId.value !== null)
+
+  /** Whether this browser can take part in calls at all. */
+  function supported(): boolean {
+    return typeof window !== 'undefined' && 'RTCPeerConnection' in window
   }
 
-  function onFeeds() {
-    watchFeeds(call.value)
-    bump()
-  }
+  // --- Tones --------------------------------------------------------------
 
   function tones() {
-    const p = phase.value
-    if (p === 'ringing') startTone('ring')
-    else if (p === 'calling') startTone('ringback')
+    if (incoming.value && !joined.value) startTone('ring')
+    else if (ringingOut.value) startTone('ringback')
     else stopTone()
   }
 
-  function onState(state: CallState) {
-    if (state === CallState.Connected) connected = true
+  // --- Who is in which room's call ----------------------------------------
+
+  function refresh(rid: string) {
+    const s = watched.get(rid)
+    const n = s ? countMembers(s) : 0
+    const next = { ...active.value }
+    if (n > 0) next[rid] = n
+    else delete next[rid]
+    active.value = next
+    // The caller gave up, or someone else answered for you.
+    if (incoming.value?.roomId === rid && n === 0) clearIncoming()
+    // Someone came: stop ringing them.
+    if (rid === roomId.value && ringingOut.value && n > 1) stopRingingOut()
+  }
+
+  function watchSession(rid: string, s: MatrixRTCSession) {
+    if (watched.get(rid) === s) return
+    watched.get(rid)?.off(MatrixRTCSessionEvent.MembershipsChanged, onMemberships)
+    watched.set(rid, s)
+    s.on(MatrixRTCSessionEvent.MembershipsChanged, onMemberships)
+    refresh(rid)
+    // Memberships are parsed asynchronously: a call already going on when
+    // the room loads only shows up once that is done.
+    void s.initialMembershipCalculated.then(() => {
+      refresh(rid)
+      bump()
+    })
+  }
+
+  /** Follows the call of every room the client knows. */
+  function watchRoom(room: MatrixRoom) {
+    if (!attached) return
+    watchSession(room.roomId, attached.matrixRTC.getRoomSession(room))
+  }
+
+  function watchAllRooms() {
+    for (const r of attached?.getRooms() ?? []) watchRoom(r)
+  }
+
+  function onSync(state: SyncState) {
+    if (state === SyncState.Prepared) watchAllRooms()
+  }
+
+  function onMemberships() {
+    for (const rid of watched.keys()) refresh(rid)
     bump()
-    tones()
   }
 
-  function onLength(seconds: number) {
-    length.value = seconds
+  function onSessionStarted(rid: string, s: MatrixRTCSession) {
+    watchSession(rid, s)
   }
 
-  function onError(err: CallError) {
-    // The hangup that follows says the rest.
-    if (err.code === CallErrorCode.NoUserMedia) {
-      forgetFailedMedia()
-      useToasts().error(isVideo.value ? 'Can’t use your camera or microphone. Check that the browser may use them.' : 'Can’t use your microphone. Check that the browser may use it.')
-    } else {
-      useToasts().error(`Call failed: ${errorMessage(err)}`)
+  function onSessionEnded(rid: string) {
+    refresh(rid)
+  }
+
+  // --- Ringing ------------------------------------------------------------
+
+  /**
+   * A ring notification or a decline arriving live. A ring counts when it
+   * is someone else's, still within its lifetime, mentions you (or the
+   * room), and you aren't in that call already.
+   */
+  function onTimeline(ev: MatrixEvent, room: MatrixRoom | undefined, toStart: boolean | undefined, _removed: boolean, data: IRoomTimelineData) {
+    if (toStart || !data?.liveEvent || !room) return
+    // In an encrypted room this is still ciphertext; onDecrypted has it.
+    if (ev.isEncrypted() && ev.isBeingDecrypted()) return
+    onCallEvent(ev, room)
+  }
+
+  /** An encrypted ring or decline, once it can be read: only recent ones count. */
+  function onDecrypted(ev: MatrixEvent) {
+    if (ev.isDecryptionFailure() || Date.now() - ev.getTs() > maxRingMs) return
+    const room = attached?.getRoom(ev.getRoomId() ?? '')
+    if (room) onCallEvent(ev, room)
+  }
+
+  /** Event IDs already acted on, so a ring isn't taken twice. */
+  const seen = new Set<string>()
+
+  function onCallEvent(ev: MatrixEvent, room: MatrixRoom) {
+    if (!attached) return
+    const type = ev.getType()
+    if (type !== EventType.RTCNotification && type !== EventType.RTCDecline) return
+    const id = ev.getId()
+    if (id) {
+      if (seen.has(id)) return
+      seen.add(id)
     }
-  }
-
-  function onHangup(c: MatrixCall) {
-    if (c !== call.value) return
-    endReason.value = describeHangup(c, connected)
-    bump()
-    stopTone()
-    window.clearTimeout(clearTimer)
-    clearTimer = window.setTimeout(() => {
-      if (call.value === c) release()
-    }, endedLingerMs)
-  }
-
-  function onReplaced(newCall: MatrixCall) {
-    // Both of you called at once: the SDK keeps one of the two calls.
-    show(newCall)
-  }
-
-  function listen(c: MatrixCall, on: boolean) {
-    const m = on ? 'on' : 'off'
-    c[m](CallEvent.State, onState)
-    c[m](CallEvent.FeedsChanged, onFeeds)
-    c[m](CallEvent.LocalHoldUnhold, bump)
-    c[m](CallEvent.RemoteHoldUnhold, bump)
-    c[m](CallEvent.LengthChanged, onLength)
-    c[m](CallEvent.Error, onError)
-    c[m](CallEvent.Hangup, onHangup)
-    c[m](CallEvent.Replaced, onReplaced)
-  }
-
-  function show(c: MatrixCall) {
-    if (call.value) listen(call.value, false)
-    window.clearTimeout(clearTimer)
-    call.value = c
-    connected = false
-    length.value = 0
-    endReason.value = ''
-    listen(c, true)
-    watchFeeds(c)
-    bump()
-    tones()
-  }
-
-  function release() {
-    if (call.value) listen(call.value, false)
-    watchFeeds(null)
-    window.clearTimeout(clearTimer)
-    call.value = null
-    stopTone()
-    bump()
-  }
-
-  function onIncoming(incoming: MatrixCall) {
-    if (call.value && phase.value !== 'ended') {
-      incoming.hangup(CallErrorCode.UserBusy, false)
+    const me = attached.getUserId() ?? ''
+    if (type === EventType.RTCNotification) {
+      if (ev.getSender() === me || roomId.value === room.roomId) return
+      const content = ev.getContent()
+      const mentions = content['m.mentions'] as { user_ids?: string[]; room?: boolean } | undefined
+      if (!mentions?.room && !mentions?.user_ids?.includes(me)) return
+      // A ring for a call of two; in a bigger room the banner says enough.
+      if (content.notification_type !== 'ring' && !otherOfTwo(room, me)) return
+      const sentAt = typeof content.sender_ts === 'number' ? Math.min(content.sender_ts, ev.getTs()) : ev.getTs()
+      const lifetime = Math.min(typeof content.lifetime === 'number' ? content.lifetime : ringLifetimeMs, maxRingMs)
+      const expiresAt = sentAt + lifetime
+      if (expiresAt <= Date.now()) return
+      if (joined.value) return // Busy: you see the banner when you're free.
+      const caller = room.getMember(ev.getSender() ?? '')
+      incoming.value = {
+        roomId: room.roomId,
+        eventId: ev.getId() ?? '',
+        callerId: ev.getSender() ?? '',
+        callerName: caller?.name ?? ev.getSender() ?? '',
+        callerAvatarMxc: caller?.getMxcAvatarUrl() ?? undefined,
+        roomName: room.name,
+        video: content['m.call.intent'] !== 'audio',
+        expiresAt,
+      }
+      window.clearTimeout(ringInTimer)
+      ringInTimer = window.setTimeout(clearIncoming, expiresAt - Date.now())
+      tones()
       return
     }
-    show(incoming)
-    // Hung up (or answered elsewhere) before we did anything.
-    if (incoming.callHasEnded()) onHangup(incoming)
+    if (type === EventType.RTCDecline) {
+      const target = ev.getContent()['m.relates_to']?.event_id
+      if (!target) return
+      // You declined on another of your devices: stop ringing here.
+      if (ev.getSender() === me && incoming.value?.eventId === target) {
+        clearIncoming()
+        return
+      }
+      // The one person you rang said no.
+      if (target === sentNotification && ringingOut.value && room.roomId === roomId.value) {
+        useToasts().info(`${room.getMember(ev.getSender() ?? '')?.name ?? 'They'} declined the call.`)
+        void leave()
+      }
+    }
+  }
+
+  function clearIncoming() {
+    window.clearTimeout(ringInTimer)
+    incoming.value = null
+    tones()
+  }
+
+  function stopRingingOut() {
+    window.clearTimeout(ringOutTimer)
+    ringingOut.value = false
+    tones()
+  }
+
+  function onNotificationSent(sent: { event_id: string; lifetime?: number }) {
+    sentNotification = sent.event_id
+  }
+
+  /** Answers the call ringing for you. */
+  async function answer(withVideo = true) {
+    const call = incoming.value
+    if (!call) return
+    clearIncoming()
+    await join(call.roomId, withVideo && call.video)
+  }
+
+  /** Declines the call ringing for you. The caller stops ringing when it's the two of you. */
+  async function decline() {
+    const call = incoming.value
+    if (!call) return
+    clearIncoming()
+    try {
+      await client().sendEvent(call.roomId, EventType.RTCDecline, {
+        'm.relates_to': { rel_type: RelationType.Reference, event_id: call.eventId },
+      })
+    } catch (e) {
+      console.error('call: decline: ', e)
+    }
   }
 
   /** Starts listening for calls on the running client. */
   function attach() {
     const c = client()
     if (attached === c) return
-    if (attached) attached.off(CallEventHandlerEvent.Incoming, onIncoming)
-    release()
+    if (attached) {
+      attached.matrixRTC.off(MatrixRTCSessionManagerEvents.SessionStarted, onSessionStarted)
+      attached.matrixRTC.off(MatrixRTCSessionManagerEvents.SessionEnded, onSessionEnded)
+      attached.off(ClientEvent.Room, watchRoom)
+      attached.off(ClientEvent.Sync, onSync)
+      attached.off(MatrixRoomEvent.Timeline, onTimeline)
+      attached.off(MatrixEventEvent.Decrypted, onDecrypted)
+    }
+    for (const s of watched.values()) s.off(MatrixRTCSessionEvent.MembershipsChanged, onMemberships)
+    watched.clear()
+    active.value = {}
+    clearIncoming()
+    void leave()
     attached = c
-    c.on(CallEventHandlerEvent.Incoming, onIncoming)
+    c.matrixRTC.on(MatrixRTCSessionManagerEvents.SessionStarted, onSessionStarted)
+    c.matrixRTC.on(MatrixRTCSessionManagerEvents.SessionEnded, onSessionEnded)
+    c.on(ClientEvent.Room, watchRoom)
+    c.on(ClientEvent.Sync, onSync)
+    c.on(MatrixRoomEvent.Timeline, onTimeline)
+    c.on(MatrixEventEvent.Decrypted, onDecrypted)
+    watchAllRooms()
   }
 
-  /** Whether this browser can make calls at all. */
-  function supported(): boolean {
-    return Boolean(attached?.supportsVoip())
-  }
+  // --- Joining ------------------------------------------------------------
 
-  async function place(roomId: string, invitee: string | undefined, video: boolean) {
-    if (call.value && phase.value !== 'ended') {
-      useToasts().error('You’re already in a call.')
-      return
+  /** The LiveKit JWT service to use: the one the call already uses, else this server's. */
+  async function chooseTransport(s: MatrixRTCSession, rid: string): Promise<LivekitTransport> {
+    const oldest = s.memberships[0]
+    const theirs = oldest?.getTransport(oldest)
+    if (theirs && theirs.type === 'livekit' && typeof theirs.livekit_service_url === 'string') {
+      return { type: 'livekit', livekit_service_url: theirs.livekit_service_url, livekit_alias: rid }
     }
-    const c = createNewMatrixCall(client(), roomId, { invitee })
-    if (!c) {
-      useToasts().error('This browser can’t make calls.')
-      return
-    }
-    show(c)
+    const c = client()
+    // The homeserver lists its backends (MSC4143); a server that predates
+    // the endpoint has them in its discovery document instead.
+    let foci: unknown
     try {
-      if (video) await c.placeVideoCall()
-      else await c.placeVoiceCall()
-    } catch (e) {
-      // The SDK has hung up already and said why through CallEvent.Error.
-      console.error('call: ', e)
+      foci = await c._unstable_getRTCTransports()
+    } catch {
+      const res = await fetch(`${c.baseUrl.replace(/\/$/, '')}/.well-known/matrix/client`)
+      if (!res.ok) throw new Error('This server doesn’t offer calls.')
+      const doc = (await res.json()) as Record<string, unknown>
+      foci = doc['org.matrix.msc4143.rtc_foci']
     }
+    const focus = Array.isArray(foci)
+      ? (foci as Record<string, unknown>[]).find((f) => f?.type === 'livekit' && typeof f.livekit_service_url === 'string')
+      : undefined
+    if (!focus) throw new Error('This server doesn’t offer calls.')
+    return { type: 'livekit', livekit_service_url: String(focus.livekit_service_url), livekit_alias: rid }
   }
 
-  /** Answers the ringing call; a video call with the camera off when `video` is false. */
-  async function answer(video = true) {
-    const c = call.value
-    if (!c || phase.value !== 'ringing') return
-    stopTone()
+  /** A LiveKit URL and token, from the JWT service, for an OpenID token. */
+  async function livekitToken(transport: LivekitTransport, rid: string): Promise<{ url: string; jwt: string }> {
+    const c = client()
+    const openid = await c.getOpenIdToken()
+    const res = await fetch(`${transport.livekit_service_url.replace(/\/$/, '')}/sfu/get`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ room: rid, openid_token: openid, device_id: c.getDeviceId() }),
+    })
+    if (!res.ok) throw new Error(`The call service refused to let you in (${res.status}).`)
+    const body = (await res.json()) as { url?: string; jwt?: string }
+    if (!body.url || !body.jwt) throw new Error('The call service gave an unusable answer.')
+    return { url: body.url, jwt: body.jwt }
+  }
+
+  function listen(r: Room) {
+    r.on(RoomEvent.ParticipantConnected, bump)
+      .on(RoomEvent.ParticipantDisconnected, bump)
+      .on(RoomEvent.TrackSubscribed, bump)
+      .on(RoomEvent.TrackUnsubscribed, bump)
+      .on(RoomEvent.TrackMuted, bump)
+      .on(RoomEvent.TrackUnmuted, bump)
+      .on(RoomEvent.LocalTrackPublished, bump)
+      .on(RoomEvent.LocalTrackUnpublished, bump)
+      .on(RoomEvent.ActiveSpeakersChanged, bump)
+      .on(RoomEvent.ParticipantEncryptionStatusChanged, bump)
+      .on(RoomEvent.Reconnecting, () => {
+        phase.value = 'reconnecting'
+      })
+      .on(RoomEvent.Reconnected, () => {
+        phase.value = 'connected'
+      })
+      .on(RoomEvent.Disconnected, (reason?: DisconnectReason) => {
+        if (lk.value !== r) return
+        if (reason !== DisconnectReason.CLIENT_INITIATED) useToasts().error('You were disconnected from the call.')
+        void leave()
+      })
+  }
+
+  function onEncryptionKey(key: Uint8Array, index: number, _membership: unknown, backendIdentity: string) {
+    void keyProvider?.setParticipantKey(key, index, backendIdentity)
+  }
+
+  function onMembershipError(e: unknown) {
+    useToasts().error(`The call stopped working: ${errorMessage(e)}`)
+    void leave()
+  }
+
+  /**
+   * Joins the call in a room, or starts one. Starting a call in a room of
+   * two rings the other person.
+   */
+  async function join(rid: string, withVideo: boolean) {
+    if (roomId.value === rid) return
+    if (roomId.value) await leave()
+    const c = client()
+    const room = c.getRoom(rid)
+    if (!room) return
+    if (incoming.value?.roomId === rid) clearIncoming()
+
+    roomId.value = rid
+    phase.value = 'joining'
+    video.value = withVideo
+    length.value = 0
+    sentNotification = null
+    bump()
+    const s = c.matrixRTC.getRoomSession(room)
+    watchSession(rid, s)
+    session = s
+    const e2ee = room.hasEncryptionStateEvent()
+    encrypted.value = e2ee
+    const me = c.getUserId()!
+    const ringing = countMembers(s) === 0 && otherOfTwo(room, me) !== undefined
     try {
-      await c.answer(true, video && c.type === CallType.Video)
+      if (e2ee && !isE2EESupported()) {
+        throw new Error('This browser can’t encrypt call media, and this room is end-to-end encrypted.')
+      }
+      const transport = await chooseTransport(s, rid)
+      const { url, jwt } = await livekitToken(transport, rid)
+      if (roomId.value !== rid) return
+
+      keyProvider = e2ee ? new MatrixKeyProvider() : null
+      const r = new Room({
+        adaptiveStream: true,
+        dynacast: true,
+        e2ee: keyProvider
+          ? { keyProvider, worker: new Worker(new URL('livekit-client/e2ee-worker', import.meta.url), { type: 'module' }) }
+          : undefined,
+      })
+      lk.value = r
+      listen(r)
+      if (keyProvider) await r.setE2EEEnabled(true)
+
+      // The Matrix side: announce ourselves (and schedule our own delayed
+      // leave), ring the other person if we're the first, and in an
+      // encrypted room start sharing media keys.
+      const deviceId = c.getDeviceId()!
+      s.on(MatrixRTCSessionEvent.EncryptionKeyChanged, onEncryptionKey)
+      s.on(MatrixRTCSessionEvent.MembershipManagerError, onMembershipError)
+      s.on(MatrixRTCSessionEvent.DidSendCallNotification, onNotificationSent)
+      s.join({ userId: me, deviceId, memberId: `${me}:${deviceId}` }, [transport], {
+        manageMediaKeys: e2ee,
+        callIntent: withVideo ? 'video' : 'audio',
+        notificationType: ringing ? 'ring' : countMembers(s) === 0 ? 'notification' : undefined,
+        notificationLifetimeMs: ringLifetimeMs,
+      })
+      if (e2ee) s.reemitEncryptionKeys()
+      if (ringing) {
+        ringingOut.value = true
+        ringOutTimer = window.setTimeout(() => {
+          if (!ringingOut.value || roomId.value !== rid) return
+          useToasts().info('No answer.')
+          void leave()
+        }, ringLifetimeMs)
+        tones()
+      }
+
+      await r.connect(url, jwt)
+      if (roomId.value !== rid) {
+        await r.disconnect()
+        return
+      }
+      phase.value = 'connected'
+      window.clearInterval(lengthTimer)
+      const since = Date.now()
+      lengthTimer = window.setInterval(() => (length.value = Math.floor((Date.now() - since) / 1000)), 1000)
+      bump()
+      try {
+        await r.localParticipant.setMicrophoneEnabled(true)
+      } catch (e) {
+        useToasts().error(`Can’t use your microphone: ${errorMessage(e)}`)
+      }
+      if (withVideo) {
+        try {
+          await r.localParticipant.setCameraEnabled(true)
+        } catch (e) {
+          useToasts().error(`Can’t use your camera: ${errorMessage(e)}`)
+        }
+      }
+      // Browsers only play audio after a gesture; joining was one.
+      await r.startAudio().catch(() => undefined)
+      bump()
     } catch (e) {
       console.error('call: ', e)
+      useToasts().error(errorMessage(e))
+      await leave()
     }
   }
 
-  function decline() {
-    const c = call.value
-    if (!c) return
-    if (phase.value === 'ringing') c.reject()
-    else hangup()
-  }
-
-  function hangup() {
-    const c = call.value
-    if (!c) return
-    if (phase.value === 'ended') release()
-    else c.hangup(CallErrorCode.UserHangup, false)
-  }
-
-  async function toggleMic() {
-    const c = call.value
-    if (!c) return
-    await c.setMicrophoneMuted(!c.isMicrophoneMuted())
+  /** Leaves the call this client is in. */
+  async function leave() {
+    const s = session
+    const r = lk.value
+    session = null
+    lk.value = null
+    keyProvider = null
+    sentNotification = null
+    window.clearInterval(lengthTimer)
+    stopRingingOut()
+    const rid = roomId.value
+    roomId.value = null
+    phase.value = null
     bump()
+    if (s) {
+      s.off(MatrixRTCSessionEvent.EncryptionKeyChanged, onEncryptionKey)
+      s.off(MatrixRTCSessionEvent.MembershipManagerError, onMembershipError)
+      s.off(MatrixRTCSessionEvent.DidSendCallNotification, onNotificationSent)
+    }
+    await Promise.allSettled([r?.disconnect(), s?.leave(5000)])
+    if (rid) refresh(rid)
   }
 
-  async function toggleCamera() {
-    const c = call.value
-    if (!c) return
-    await c.setLocalVideoMuted(!c.isLocalVideoMuted())
-    bump()
-  }
+  // --- What the view shows ------------------------------------------------
 
-  async function toggleScreenShare() {
-    const c = call.value
-    if (!c) return
+  const tiles = computed<Tile[]>(() => {
+    void version.value
+    const r = lk.value
+    const rid = roomId.value
+    if (!r || !rid) return []
+    const room = attached?.getRoom(rid)
+    const speakers = new Set(r.activeSpeakers.map((p) => p.identity))
+    const tile = (p: Participant, local: boolean): Tile => {
+      const userId = identityUser(p.identity)
+      const member = room?.getMember(userId)
+      const cam = p.getTrackPublication(Track.Source.Camera)
+      const screen = p.getTrackPublication(Track.Source.ScreenShare)
+      const mic = p.getTrackPublication(Track.Source.Microphone)
+      return {
+        id: p.identity,
+        userId,
+        name: member?.name ?? userId,
+        avatarMxc: member?.getMxcAvatarUrl() ?? undefined,
+        local,
+        camera: cam?.track && !cam.isMuted ? cam.track : undefined,
+        screen: screen?.track && !screen.isMuted ? screen.track : undefined,
+        audio: !local && mic?.track ? mic.track : undefined,
+        micMuted: !mic || mic.isMuted,
+        speaking: speakers.has(p.identity),
+      }
+    }
+    return [tile(r.localParticipant, true), ...[...r.remoteParticipants.values()].map((p) => tile(p, false))]
+  })
+
+  /** The room's name, or for a call of two, the other person's. */
+  const title = computed(() => {
+    void version.value
+    const rid = roomId.value
+    const room = rid ? attached?.getRoom(rid) : undefined
+    if (!room) return ''
+    const other = otherOfTwo(room, attached?.getUserId() ?? '')
+    return other ? (room.getMember(other)?.name ?? room.name) : room.name
+  })
+
+  /** The person you're calling, for the "ringing" screen of a call of two. */
+  const callee = computed(() => {
+    void version.value
+    const rid = roomId.value
+    const room = rid ? attached?.getRoom(rid) : undefined
+    const other = room ? otherOfTwo(room, attached?.getUserId() ?? '') : undefined
+    const member = other ? room?.getMember(other) : undefined
+    return member ? { name: member.name, avatarMxc: member.getMxcAvatarUrl() ?? undefined } : null
+  })
+
+  const micOn = computed(() => {
+    void version.value
+    return lk.value?.localParticipant.isMicrophoneEnabled ?? false
+  })
+  const cameraOn = computed(() => {
+    void version.value
+    return lk.value?.localParticipant.isCameraEnabled ?? false
+  })
+  const sharing = computed(() => {
+    void version.value
+    return lk.value?.localParticipant.isScreenShareEnabled ?? false
+  })
+
+  async function toggle(what: 'mic' | 'camera' | 'screen') {
+    const p = lk.value?.localParticipant
+    if (!p) return
     try {
-      await c.setScreensharingEnabled(!c.isScreensharing())
+      if (what === 'mic') await p.setMicrophoneEnabled(!p.isMicrophoneEnabled)
+      else if (what === 'camera') await p.setCameraEnabled(!p.isCameraEnabled)
+      else await p.setScreenShareEnabled(!p.isScreenShareEnabled, { audio: true })
     } catch (e) {
-      useToasts().error(`Can’t share your screen: ${errorMessage(e)}`)
+      // Cancelling the browser's screen picker is not an error worth a toast.
+      if (!(e instanceof DOMException && e.name === 'NotAllowedError' && what === 'screen')) {
+        useToasts().error(errorMessage(e))
+      }
     }
     bump()
   }
 
-  function toggleHold() {
-    const c = call.value
-    if (!c) return
-    c.setRemoteOnHold(!c.isRemoteOnHold())
-    bump()
+  /** How many people are in a room's call; 0 when there is none. */
+  function inCall(rid: string): number {
+    return active.value[rid] ?? 0
   }
 
   return {
-    call,
-    version,
-    length,
-    endReason,
+    roomId,
     phase,
-    isVideo,
-    peerName,
-    peerAvatarMxc,
-    attach,
+    version,
+    active,
+    joined,
+    encrypted,
+    video,
+    length,
+    ringingOut,
+    incoming,
+    tiles,
+    title,
+    callee,
+    micOn,
+    cameraOn,
+    sharing,
     supported,
-    place,
+    attach,
+    join,
+    leave,
     answer,
     decline,
-    hangup,
-    toggleMic,
-    toggleCamera,
-    toggleScreenShare,
-    toggleHold,
+    toggle,
+    inCall,
   }
 })
-
-/**
- * The SDK's media handler queues every request for the microphone and
- * camera behind the previous one, and never lets go of a request that
- * failed: once you said no, every later call would fail the same way until
- * the page reloads. Dropping the failed request lets the next call ask again.
- */
-function forgetFailedMedia() {
-  const handler = client().getMediaHandler() as unknown as { getMediaStreamPromise?: Promise<MediaStream> }
-  handler.getMediaStreamPromise = undefined
-}
-
-function describeHangup(c: MatrixCall, connected: boolean): string {
-  switch (c.hangupReason) {
-    case CallErrorCode.UserBusy:
-      return 'Busy'
-    case CallErrorCode.InviteTimeout:
-      return c.direction === CallDirection.Inbound ? 'Missed call' : 'No answer'
-    case CallErrorCode.AnsweredElsewhere:
-      return 'Answered in another session'
-    case CallErrorCode.IceFailed:
-      return 'Couldn’t connect'
-    case CallErrorCode.NoUserMedia:
-      return 'No microphone or camera'
-    case CallErrorCode.Replaced:
-      return 'Call replaced'
-    default:
-      if (!connected) {
-        const outbound = c.direction === CallDirection.Outbound
-        if (c.hangupParty === CallParty.Remote) return outbound ? 'Declined' : 'Missed call'
-        return outbound ? 'Call cancelled' : 'Declined'
-      }
-      return 'Call ended'
-  }
-}

apps/web/src/stores/chat.ts

@@ -60,8 +60,9 @@ export function describeEvent(ev: MatrixEvent, me: string, room: Room, withSende
   // own placeholder text: never show that.
   if (ev.isDecryptionFailure()) return withSender ? `${senderName(ev, me, room)}: Can’t decrypt this message` : 'Can’t decrypt this message'
   if (type === EventType.RoomMessageEncrypted) return 'Encrypted message'
-  if (type === EventType.CallInvite) {
-    const text = isVideoCallInvite(ev) ? 'Video call' : 'Voice call'
+  if (type === EventType.CallInvite || type === EventType.RTCNotification) {
+    const video = type === EventType.CallInvite ? isVideoCallInvite(ev) : ev.getContent()['m.call.intent'] !== 'audio'
+    const text = video ? 'Video call' : 'Voice call'
     return withSender ? `${senderName(ev, me, room)}: ${text}` : text
   }
   if (type !== EventType.RoomMessage && type !== EventType.Sticker) return null