Group voice and video calls with MatrixRTC and LiveKit

Every call in the web client now goes through MatrixRTC with a LiveKit
SFU, the model Element Call and Element X use; the legacy 1:1 m.call.*
calls are gone. In a room of two the other person is rung and can answer
or decline; in a bigger room a banner offers to join the call. Calls show
a grid of participant tiles with mute, camera, screen sharing and leave,
and media is end-to-end encrypted in encrypted rooms.

Server:
- MSC4140 delayed events (send/state with a delay, plus the
  delayed_events list/restart/cancel/send API), stored and scheduled, so
  a member whose client drops off is taken out of the call.
- POST /user/{userId}/openid/request_token, which lk-jwt-service uses to
  hand out LiveKit tokens.
- org.matrix.msc4143.rtc_foci in /.well-known/matrix/client, configured
  under client_api.rtc.
- Room presets let members send call membership state.
- Migration 0014 for both SQL dialects.

Deploy: LiveKit and lk-jwt-service in compose (calls.yml) and the helm
chart. Docs: docs/calls.md, configuration and deployment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

apps/web/src/components/CallTile.vue

@@ -0,0 +1,58 @@
+<script setup lang="ts">
+import type { Track } from 'livekit-client'
+import { computed, onBeforeUnmount, ref, watch } from 'vue'
+import { WIcon } from '@warp/ui'
+import MxcAvatar from '@/components/MxcAvatar.vue'
+import type { Tile } from '@/stores/call'
+
+// One person in a group call: their screen or camera, else their avatar,
+// with their name, a muted microphone, and a ring while they speak.
+
+const props = defineProps<{ tile: Tile; compact?: boolean }>()
+
+const videoEl = ref<HTMLVideoElement>()
+const audioEl = ref<HTMLAudioElement>()
+
+const shown = computed(() => props.tile.screen ?? props.tile.camera)
+// Your own camera is mirrored, the way a mirror (and every call app) shows you.
+const mirrored = computed(() => props.tile.local && !props.tile.screen)
+
+function bind(el: HTMLMediaElement | undefined, track: Track | undefined, old: Track | undefined) {
+  if (!el) return
+  if (old && old !== track) old.detach(el)
+  if (track) track.attach(el)
+}
+
+watch([videoEl, shown], ([el, track], [, old]) => bind(el, track, old), { flush: 'post', immediate: true })
+watch([audioEl, () => props.tile.audio], ([el, track], [, old]) => bind(el, track, old), { flush: 'post', immediate: true })
+
+onBeforeUnmount(() => {
+  if (videoEl.value) shown.value?.detach(videoEl.value)
+  if (audioEl.value) props.tile.audio?.detach(audioEl.value)
+})
+</script>
+
+<template>
+  <div
+    :class="[
+      'relative flex min-h-0 items-center justify-center overflow-hidden rounded-md bg-[#1d2126] transition-shadow',
+      tile.speaking ? 'shadow-[0_0_0_3px_#3fb68b]' : '',
+    ]"
+  >
+    <audio v-if="tile.audio" ref="audioEl" autoplay class="hidden" />
+    <video
+      v-show="shown"
+      ref="videoEl"
+      autoplay
+      playsinline
+      muted
+      :class="['h-full w-full', tile.screen ? 'object-contain' : 'object-cover', mirrored && '-scale-x-100']"
+    />
+    <MxcAvatar v-if="!shown" :name="tile.name" :mxc="tile.avatarMxc" :size="compact ? 48 : 88" />
+    <div class="absolute bottom-2 left-2 flex max-w-[calc(100%-1rem)] items-center gap-1 rounded-sm bg-[#000000a6] px-2 py-0.5">
+      <WIcon v-if="tile.micMuted" name="mic-off" :size="14" label="Muted" class="shrink-0 text-[#ff8a80]" />
+      <WIcon v-if="tile.screen" name="screen-share" :size="14" label="Sharing their screen" class="shrink-0" />
+      <span class="caption truncate text-[#eceeee]">{{ tile.local ? `${tile.name} (you)` : tile.name }}</span>
+    </div>
+  </div>
+</template>

apps/web/src/components/CallView.vue

@@ -1,108 +1,74 @@
 <script setup lang="ts">
-import type { CallFeed } from 'matrix-js-sdk/lib/webrtc/callFeed'
-import { computed, onBeforeUnmount, ref, watch, watchEffect } from 'vue'
+import { computed, onBeforeUnmount, ref, watch } from 'vue'
 import { useRouter } from 'vue-router'
-import { WIcon } from '@warp/ui'
+import { WIcon, WSpinner } from '@warp/ui'
 import type { IconName } from '@warp/ui'
+import CallTile from '@/components/CallTile.vue'
 import MxcAvatar from '@/components/MxcAvatar.vue'
 import { formatDuration } from '@/lib/format'
-import { useCall } from '@/stores/call'
+import { useCall, type Tile } from '@/stores/call'
 
-// The call on screen: a ringing card for an incoming call, else a panel in
-// the corner that a video call can expand to fill the window. It sits above
-// every view, so a call goes on while you move between rooms.
+// The call on screen: a ringing card for a call coming in, else the call
+// you're in. A call of two shows the other person large and you in the
+// corner; a bigger one is a grid. The panel shrinks to the corner so the
+// call goes on while you read and write in your rooms, and sits above
+// every view.
 
 const call = useCall()
 const router = useRouter()
-const expanded = ref(false)
-
-function feedVideo(feed: CallFeed | undefined): MediaStream | undefined {
-  if (!feed || feed.isVideoMuted() || !feed.stream.getVideoTracks().length) return undefined
-  return feed.stream
-}
-
-const view = computed(() => {
-  void call.version
-  const c = call.call
-  if (!c) return null
-  const remoteCamera = feedVideo(c.remoteUsermediaFeed)
-  const remoteScreen = c.remoteScreensharingStream
-  const localCamera = feedVideo(c.localUsermediaFeed)
-  const localScreen = c.localScreensharingStream
-  // Their screen beats their camera on the big stage; whatever is left of
-  // theirs or yours goes in the small picture.
-  const main = remoteScreen ?? remoteCamera
-  const pip = remoteScreen && remoteCamera ? remoteCamera : (localScreen ?? localCamera)
-  return {
-    main,
-    pip,
-    pipMirrored: pip !== undefined && pip === localCamera,
-    remoteAudio: c.remoteUsermediaStream,
-    micMuted: c.isMicrophoneMuted(),
-    cameraOn: Boolean(localCamera),
-    sharing: c.isScreensharing(),
-    remoteOnHold: c.isRemoteOnHold(),
-    localOnHold: c.isLocalOnHold(),
-    roomId: c.roomId,
-  }
-})
-
-const stage = computed(() => call.phase !== 'ringing' && Boolean(view.value?.main || view.value?.pip || call.isVideo))
-const canShare = typeof navigator !== 'undefined' && Boolean(navigator.mediaDevices?.getDisplayMedia)
-
-const status = computed(() => {
-  const v = view.value
-  switch (call.phase) {
-    case 'ringing':
-      return call.isVideo ? 'Incoming video call' : 'Incoming voice call'
-    case 'calling':
-      return 'Calling…'
-    case 'connecting':
-      return 'Connecting…'
-    case 'connected':
-      if (v?.localOnHold) return 'You’re on hold'
-      if (v?.remoteOnHold) return 'On hold'
-      return formatDuration(call.length)
-    case 'ended':
-      return call.endReason
-    default:
-      return ''
-  }
-})
-
-// Streams go to the media elements by hand: srcObject is not an attribute.
-const mainEl = ref<HTMLVideoElement>()
-const pipEl = ref<HTMLVideoElement>()
-const audioEl = ref<HTMLAudioElement>()
-
-function bind(el: HTMLMediaElement | undefined, stream: MediaStream | undefined) {
-  if (!el || el.srcObject === (stream ?? null)) return
-  el.srcObject = stream ?? null
-  if (stream) void el.play().catch(() => undefined)
-}
-watchEffect(() => bind(mainEl.value, view.value?.main), { flush: 'post' })
-watchEffect(() => bind(pipEl.value, view.value?.pip), { flush: 'post' })
-watchEffect(() => bind(audioEl.value, view.value?.remoteAudio), { flush: 'post' })
+const expanded = ref(true)
 
 watch(
-  () => call.call,
-  (c) => {
-    if (!c) expanded.value = false
+  () => call.roomId,
+  (rid) => {
+    // Video calls and group calls open full size; a voice call between
+    // two people starts as the panel in the corner.
+    if (rid) expanded.value = call.video || !call.callee
   },
 )
 
-function onKey(e: KeyboardEvent) {
-  if (e.key === 'Escape' && expanded.value) expanded.value = false
-}
-window.addEventListener('keydown', onKey)
-onBeforeUnmount(() => window.removeEventListener('keydown', onKey))
+const remotes = computed(() => call.tiles.filter((t) => !t.local))
+const self = computed(() => call.tiles.find((t) => t.local))
+/** A call of two: the other person on the stage, you in the corner. */
+const pair = computed(() => remotes.value.length === 1)
+/** Ringing them, nobody here yet. */
+const calling = computed(() => call.ringingOut && remotes.value.length === 0)
 
-function openRoom() {
-  const roomId = view.value?.roomId
-  if (!roomId) return
-  expanded.value = false
-  router.push({ name: 'room', params: { roomId } })
-}
+// Whoever shares their screen takes the stage; otherwise the person
+// speaking comes first, so the panel in the corner shows them.
+const ordered = computed(() => {
+  const rank = (t: Tile) => (t.screen ? 0 : t.speaking && !t.local ? 1 : t.local ? 3 : 2)
+  return [...call.tiles].sort((a, b) => rank(a) - rank(b))
+})
+const shown = computed(() => (expanded.value ? ordered.value : ordered.value.slice(0, 1)))
+// The rest still have to be heard while the panel shows only one.
+const hidden = computed(() => (expanded.value ? [] : ordered.value.slice(1).filter((t) => t.audio)))
+
+const columns = computed(() => {
+  const n = shown.value.length
+  if (n <= 1) return 'grid-cols-1'
+  if (n <= 4) return 'grid-cols-1 sm:grid-cols-2'
+  if (n <= 9) return 'grid-cols-2 lg:grid-cols-3'
+  return 'grid-cols-2 md:grid-cols-3 xl:grid-cols-4'
+})
+
+const status = computed(() => {
+  switch (call.phase) {
+    case 'joining':
+      return calling.value ? 'Calling…' : 'Joining…'
+    case 'reconnecting':
+      return 'Reconnecting…'
+    default: {
+      if (calling.value) return 'Calling…'
+      const time = formatDuration(call.length)
+      if (call.callee) return time
+      const n = call.tiles.length
+      return n === 1 ? `Only you so far · ${time}` : `${n} people · ${time}`
+    }
+  }
+})
+
+const canShare = typeof navigator !== 'undefined' && Boolean(navigator.mediaDevices?.getDisplayMedia)
 
 interface Control {
   icon: IconName
@@ -113,23 +79,28 @@ interface Control {
 }
 
 const controls = computed<Control[]>(() => {
-  const v = view.value
-  if (!v) return []
-  if (call.phase === 'ringing') {
-    const list: Control[] = [{ icon: 'phone-off', label: 'Decline', action: call.decline, tone: 'danger' }]
-    if (call.isVideo) list.push({ icon: 'phone', label: 'Answer without video', action: () => call.answer(false) })
-    list.push({ icon: call.isVideo ? 'video' : 'phone', label: 'Answer', action: () => call.answer(), tone: 'accept' })
-    return list
-  }
-  if (call.phase === 'ended') return [{ icon: 'close', label: 'Close', action: call.hangup }]
   const list: Control[] = [
-    { icon: v.micMuted ? 'mic-off' : 'mic', label: v.micMuted ? 'Unmute' : 'Mute', action: call.toggleMic, on: v.micMuted },
-    { icon: v.cameraOn ? 'video' : 'video-off', label: v.cameraOn ? 'Turn camera off' : 'Turn camera on', action: call.toggleCamera, on: !v.cameraOn && call.isVideo },
+    { icon: call.micOn ? 'mic' : 'mic-off', label: call.micOn ? 'Mute' : 'Unmute', action: () => call.toggle('mic'), on: !call.micOn },
+    {
+      icon: call.cameraOn ? 'video' : 'video-off',
+      label: call.cameraOn ? 'Turn camera off' : 'Turn camera on',
+      action: () => call.toggle('camera'),
+      on: !call.cameraOn,
+    },
   ]
   if (canShare && call.phase === 'connected') {
-    list.push({ icon: 'screen-share', label: v.sharing ? 'Stop sharing' : 'Share screen', action: call.toggleScreenShare, on: v.sharing })
+    list.push({ icon: 'screen-share', label: call.sharing ? 'Stop sharing' : 'Share screen', action: () => call.toggle('screen'), on: call.sharing })
   }
-  list.push({ icon: 'phone-off', label: 'Hang up', action: call.hangup, tone: 'danger' })
+  list.push({ icon: 'phone-off', label: calling.value ? 'Cancel call' : 'Leave call', action: () => void call.leave(), tone: 'danger' })
+  return list
+})
+
+const ringControls = computed<Control[]>(() => {
+  const c = call.incoming
+  if (!c) return []
+  const list: Control[] = [{ icon: 'phone-off', label: 'Decline', action: () => void call.decline(), tone: 'danger' }]
+  if (c.video) list.push({ icon: 'phone', label: 'Answer without video', action: () => void call.answer(false) })
+  list.push({ icon: c.video ? 'video' : 'phone', label: 'Answer', action: () => void call.answer(), tone: 'accept' })
   return list
 })
 
@@ -139,37 +110,70 @@ function controlClass(c: Control): string {
   if (c.on) return 'bg-[#eceeee] text-[#111418] hover:bg-[#ffffff]'
   return 'bg-[#ffffff1f] text-[#eceeee] hover:bg-[#ffffff33]'
 }
+
+function openRoom() {
+  const roomId = call.roomId
+  if (!roomId) return
+  expanded.value = false
+  router.push({ name: 'room', params: { roomId } })
+}
+
+function onKey(e: KeyboardEvent) {
+  if (e.key === 'Escape' && expanded.value) expanded.value = false
+}
+window.addEventListener('keydown', onKey)
+onBeforeUnmount(() => window.removeEventListener('keydown', onKey))
 </script>
 
 <template>
   <section
-    v-if="call.call && view"
+    v-if="call.incoming && !call.joined"
+    class="fixed right-4 bottom-4 left-4 z-40 flex flex-col overflow-hidden rounded-lg bg-[#111418] text-[#eceeee] shadow-float sm:left-auto sm:w-[360px]"
+    :aria-label="`Call from ${call.incoming.callerName}`"
+    role="alertdialog"
+  >
+    <div class="flex items-center gap-3 px-4 pt-4">
+      <MxcAvatar :name="call.incoming.callerName" :mxc="call.incoming.callerAvatarMxc" :size="48" />
+      <div class="min-w-0 flex-1">
+        <span class="body-strong block truncate">{{ call.incoming.callerName }}</span>
+        <span class="small block truncate text-[#eceeeeb3]">{{ call.incoming.video ? 'Incoming video call' : 'Incoming voice call' }}</span>
+      </div>
+    </div>
+    <div class="flex items-center justify-center gap-3 px-4 pt-3 pb-4">
+      <button
+        v-for="c in ringControls"
+        :key="c.label"
+        type="button"
+        :class="['flex h-12 w-12 cursor-pointer items-center justify-center rounded-full border-0 transition-colors', controlClass(c)]"
+        :aria-label="c.label"
+        :title="c.label"
+        @click="c.action()"
+      >
+        <WIcon :name="c.icon" :size="22" />
+      </button>
+    </div>
+  </section>
+
+  <section
+    v-if="call.roomId"
     :class="[
       'fixed z-40 flex flex-col overflow-hidden bg-[#111418] text-[#eceeee] shadow-float',
       expanded ? 'inset-0' : 'right-4 bottom-4 left-4 rounded-lg sm:left-auto sm:w-[360px]',
     ]"
-    :aria-label="`Call with ${call.peerName}`"
-    :role="call.phase === 'ringing' ? 'alertdialog' : 'region'"
+    :aria-label="`Call: ${call.title}`"
+    role="region"
   >
-    <audio ref="audioEl" autoplay class="hidden" />
-
-    <div v-if="stage" :class="['relative bg-[#000000]', expanded ? 'min-h-0 flex-1' : 'aspect-video']">
-      <video v-show="view.main" ref="mainEl" autoplay playsinline muted class="h-full w-full object-contain" />
-      <div v-if="!view.main" class="absolute inset-0 flex items-center justify-center">
-        <MxcAvatar :name="call.peerName" :mxc="call.peerAvatarMxc" :size="expanded ? 120 : 72" />
-      </div>
-      <video
-        v-show="view.pip"
-        ref="pipEl"
-        autoplay
-        playsinline
-        muted
-        :class="['absolute right-3 bottom-3 w-1/4 min-w-24 rounded-md bg-[#000000] object-cover shadow-float', view.pipMirrored && '-scale-x-100']"
-      />
+    <div class="flex items-center gap-3 px-4 pt-3">
+      <button type="button" class="min-w-0 flex-1 cursor-pointer border-0 bg-transparent p-0 text-left text-[#eceeee]" @click="openRoom">
+        <span class="body-strong flex items-center gap-2 truncate">
+          {{ call.title }}
+          <WIcon v-if="call.encrypted" name="lock" :size="14" label="End-to-end encrypted" class="shrink-0 text-[#3fb68b]" />
+        </span>
+        <span class="small block truncate text-[#eceeeeb3] tabular-nums" aria-live="polite">{{ status }}</span>
+      </button>
       <button
-        v-if="call.isVideo || view.main"
         type="button"
-        class="absolute top-2 right-2 flex h-9 w-9 cursor-pointer items-center justify-center rounded-full border-0 bg-[#00000080] text-[#eceeee] hover:bg-[#000000b3]"
+        class="flex h-9 w-9 cursor-pointer items-center justify-center rounded-full border-0 bg-[#ffffff1f] text-[#eceeee] hover:bg-[#ffffff33]"
         :aria-label="expanded ? 'Shrink' : 'Expand'"
         :title="expanded ? 'Shrink' : 'Expand'"
         @click="expanded = !expanded"
@@ -178,12 +182,34 @@ function controlClass(c: Control): string {
       </button>
     </div>
 
-    <div class="flex items-center gap-3 px-4 pt-4">
-      <MxcAvatar v-if="!stage" :name="call.peerName" :mxc="call.peerAvatarMxc" :size="48" />
-      <button type="button" class="min-w-0 flex-1 cursor-pointer border-0 bg-transparent p-0 text-left text-[#eceeee]" @click="openRoom">
-        <span class="body-strong block truncate">{{ call.peerName }}</span>
-        <span class="small block truncate text-[#eceeeeb3] tabular-nums" aria-live="polite">{{ status }}</span>
-      </button>
+    <div :class="['relative px-4 pt-3', expanded ? 'min-h-0 flex-1' : '']">
+      <!-- Joining, or ringing someone who hasn't come yet. -->
+      <div
+        v-if="call.phase === 'joining' || calling"
+        :class="['flex flex-col items-center justify-center gap-4', expanded ? 'h-full' : 'aspect-video']"
+      >
+        <MxcAvatar v-if="call.callee" :name="call.callee.name" :mxc="call.callee.avatarMxc" :size="expanded ? 120 : 72" />
+        <WSpinner v-else :size="32" />
+      </div>
+
+      <!-- Two of you: them on the stage, you in the corner. -->
+      <div v-else-if="pair && remotes[0]" :class="['relative', expanded ? 'h-full' : 'aspect-video']">
+        <CallTile :tile="remotes[0]" :compact="!expanded" class="h-full w-full" />
+        <CallTile
+          v-if="self"
+          :tile="self"
+          compact
+          class="absolute right-3 bottom-3 aspect-video w-1/4 min-w-28 shadow-float"
+        />
+      </div>
+
+      <!-- More: a grid, or whoever is speaking when shrunk. -->
+      <template v-else>
+        <div :class="['grid gap-2', expanded ? `h-full auto-rows-fr ${columns}` : 'aspect-video grid-cols-1']">
+          <CallTile v-for="t in shown" :key="t.id" :tile="t" :compact="!expanded" />
+        </div>
+        <CallTile v-for="t in hidden" :key="t.id" :tile="t" class="hidden" />
+      </template>
     </div>
 
     <div class="flex items-center justify-center gap-3 px-4 pt-3 pb-4">
@@ -193,7 +219,7 @@ function controlClass(c: Control): string {
         type="button"
         :class="['flex h-12 w-12 cursor-pointer items-center justify-center rounded-full border-0 transition-colors', controlClass(c)]"
         :aria-label="c.label"
-        :aria-pressed="c.on === undefined || c.tone ? undefined : c.on"
+        :aria-pressed="c.tone ? undefined : c.on"
         :title="c.label"
         @click="c.action()"
       >