Files
Warp/apps/web/src/matrix/timeline.ts
T
tajniak81andClaude Opus 5.5 76dfc98a86
CI / Test (ubuntu-latest) (push) Canceled after 0s
CI / Test (windows-latest) (push) Canceled after 0s
CI / Storage tests on PostgreSQL (push) Canceled after 0s
CI / Complement (federation off) (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
CI / Smoke test the binary (push) Canceled after 0s
CI / Admin panel (push) Canceled after 0s
CI / Web client (push) Canceled after 0s
Group voice and video calls with MatrixRTC and LiveKit
Every call in the web client now goes through MatrixRTC with a LiveKit
SFU, the model Element Call and Element X use; the legacy 1:1 m.call.*
calls are gone. In a room of two the other person is rung and can answer
or decline; in a bigger room a banner offers to join the call. Calls show
a grid of participant tiles with mute, camera, screen sharing and leave,
and media is end-to-end encrypted in encrypted rooms.

Server:
- MSC4140 delayed events (send/state with a delay, plus the
  delayed_events list/restart/cancel/send API), stored and scheduled, so
  a member whose client drops off is taken out of the call.
- POST /user/{userId}/openid/request_token, which lk-jwt-service uses to
  hand out LiveKit tokens.
- org.matrix.msc4143.rtc_foci in /.well-known/matrix/client, configured
  under client_api.rtc.
- Room presets let members send call membership state.
- Migration 0014 for both SQL dialects.

Deploy: LiveKit and lk-jwt-service in compose (calls.yml) and the helm
chart. Docs: docs/calls.md, configuration and deployment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-18 19:33:29 +02:00

442 lines
16 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import {
Direction,
EventStatus,
EventType,
MsgType,
type MatrixClient,
type MatrixEvent,
type Room,
} from 'matrix-js-sdk'
import { DecryptionFailureCode } from 'matrix-js-sdk/lib/crypto-api'
import type { RoomMessageEventContent } from 'matrix-js-sdk/lib/types'
import type { MessageStatus } from '@warp/ui'
import { sameDay } from '@/lib/format'
import type { EncryptedFile } from './attachments'
import type { MediaSource } from './media'
import { shieldOf, type Shield } from './trust'
// Turns a room's live timeline into what the timeline view draws: messages
// grouped by sender, day separators and one-line notices for state changes.
// The SDK keeps the events; this is a pure projection of them.
export interface MediaInfo {
mxc?: string
/** The thumbnail the sender uploaded, when there is one. */
thumbMxc?: string
/** Set in an encrypted room: the file is ciphertext, this decrypts it. */
file?: EncryptedFile
thumbFile?: EncryptedFile
thumbMimetype?: string
mimetype?: string
size?: number
w?: number
h?: number
filename: string
}
export interface MessageItem {
kind: 'message'
key: string
eventId?: string
sender: string
senderName: string
avatarMxc?: string
own: boolean
ts: number
msgtype: string
body: string
media?: MediaInfo
edited: boolean
redacted: boolean
/** Still ciphertext: being decrypted, or it can't be (see `utd`). */
encrypted: boolean
/** Why this session can't decrypt it. */
utd?: string
/** Sent end-to-end encrypted. */
e2ee: boolean
/** Why this message may not be from who it says, when that's in doubt. */
shield?: Shield
status?: MessageStatus
/** First of a run of messages from one sender: shows the name. */
first: boolean
/** Last of a run: shows the time and status. */
last: boolean
event: MatrixEvent
}
export interface NoticeItem {
kind: 'notice'
key: string
ts: number
text: string
}
export interface DayItem {
kind: 'day'
key: string
ts: number
}
export type TimelineItem = MessageItem | NoticeItem | DayItem
// Messages from one sender this close together form one group.
const groupGapMs = 5 * 60_000
function nameOf(room: Room, userId: string): string {
return room.getMember(userId)?.name ?? userId
}
function statusOf(ev: MatrixEvent): MessageStatus | undefined {
switch (ev.status) {
case EventStatus.NOT_SENT:
return 'failed'
case EventStatus.SENDING:
case EventStatus.QUEUED:
case EventStatus.ENCRYPTING:
return 'sending'
case EventStatus.CANCELLED:
return undefined
default:
return 'sent'
}
}
/** Whether an m.call.invite offers video. */
export function isVideoCallInvite(ev: MatrixEvent): boolean {
const sdp = ev.getContent().offer?.sdp
return typeof sdp === 'string' && sdp.includes('m=video')
}
/** A one-line description of a call event, or null for the signalling in between. */
function describeCall(ev: MatrixEvent, room: Room): string | null {
const sender = nameOf(room, ev.getSender() ?? '')
switch (ev.getType()) {
case EventType.CallInvite:
return `${sender} started a ${isVideoCallInvite(ev) ? 'video' : 'voice'} call`
case EventType.CallReject:
case EventType.RTCDecline:
return `${sender} declined the call`
case EventType.CallHangup:
switch (ev.getContent().reason) {
case 'invite_timeout':
return 'The call wasn’t answered'
case 'user_busy':
return `${sender} was busy`
case 'ice_failed':
return 'The call couldn’t connect'
default:
return `${sender} ended the call`
}
default:
return null
}
}
/**
* Whether a call.member state event (MatrixRTC) says its device is in the
* call. Leaving is an empty event; the legacy form lists memberships.
*/
export function isActiveCallMember(content: Record<string, unknown>): boolean {
if (Object.keys(content).length === 0) return false
if (Array.isArray(content.memberships)) return content.memberships.length > 0
return true
}
/**
* Follows who is in a room's group call along the timeline, to say when
* it started and ended. Each device in the call has a call.member state
* event of its own; a member refreshing theirs says nothing.
*/
class CallTracker {
private readonly inCall = new Set<string>()
constructor(room: Room) {
const start = room.getLiveTimeline().getState(Direction.Backward)
for (const ev of start?.getStateEvents(EventType.GroupCallMemberPrefix) ?? []) {
if (isActiveCallMember(ev.getContent())) this.inCall.add(ev.getStateKey() ?? '')
}
}
describe(ev: MatrixEvent, room: Room): string | null {
const key = ev.getStateKey() ?? ''
const was = this.inCall.has(key)
const now = isActiveCallMember(ev.getContent())
if (was === now) return null
const sender = nameOf(room, ev.getSender() ?? '')
if (now) {
const first = this.inCall.size === 0
this.inCall.add(key)
return first ? `${sender} started a call` : `${sender} joined the call`
}
this.inCall.delete(key)
return this.inCall.size === 0 ? 'The call ended' : `${sender} left the call`
}
}
/** A one-line description of a state change, or null to leave it out. */
function describeState(ev: MatrixEvent, room: Room): string | null {
const sender = nameOf(room, ev.getSender() ?? '')
const content = ev.getContent()
const prev = ev.getPrevContent()
switch (ev.getType()) {
case EventType.RoomCreate:
return `${sender} created the room`
case EventType.RoomName:
return content.name ? `${sender} named the room ${content.name}` : `${sender} removed the room name`
case EventType.RoomTopic:
return content.topic ? `${sender} changed the topic to “${content.topic}”` : `${sender} removed the topic`
case EventType.RoomAvatar:
return `${sender} changed the room avatar`
case EventType.RoomEncryption:
return `${sender} turned on end-to-end encryption`
case EventType.RoomTombstone:
return `${sender} upgraded this room`
case EventType.RoomMember: {
const target = ev.getStateKey() ?? ''
const targetName = content.displayname ?? prev.displayname ?? target
const reason = content.reason ? `: ${content.reason}` : ''
switch (content.membership) {
case 'join':
if (prev.membership === 'join') {
if (content.displayname !== prev.displayname) {
if (!content.displayname) return `${prev.displayname ?? target} removed their display name`
if (!prev.displayname) return `${target} set their display name to ${content.displayname}`
return `${prev.displayname} changed their name to ${content.displayname}`
}
if (content.avatar_url !== prev.avatar_url) return `${targetName} changed their avatar`
return null
}
return `${targetName} joined`
case 'invite':
return `${sender} invited ${targetName}`
case 'leave':
if (target !== ev.getSender()) {
if (prev.membership === 'ban') return `${sender} unbanned ${targetName}`
if (prev.membership === 'invite') return `${sender} withdrew ${targetName}’s invitation`
return `${sender} removed ${targetName}${reason}`
}
return prev.membership === 'invite' ? `${targetName} declined the invitation` : `${targetName} left`
case 'ban':
return `${sender} banned ${targetName}${reason}`
case 'knock':
return `${targetName} asked to join`
}
return null
}
}
return null
}
/** What a message this session can't decrypt says about why, honestly. */
export function utdText(ev: MatrixEvent, cryptoRunning: boolean): string | undefined {
if (!ev.isDecryptionFailure()) return cryptoRunning ? undefined : 'End-to-end encryption didn’t start in this browser, so it can’t read this message.'
switch (ev.decryptionFailureReason) {
case DecryptionFailureCode.HISTORICAL_MESSAGE_NO_KEY_BACKUP:
case DecryptionFailureCode.HISTORICAL_MESSAGE_BACKUP_UNCONFIGURED:
return 'Sent before this session signed in. Verify this session or restore your key backup to read it.'
case DecryptionFailureCode.HISTORICAL_MESSAGE_WORKING_BACKUP:
return 'Sent before this session signed in, and its key isn’t in your key backup.'
case DecryptionFailureCode.HISTORICAL_MESSAGE_USER_NOT_JOINED:
return 'Sent before you joined this room, so you can’t read it.'
case DecryptionFailureCode.MEGOLM_KEY_WITHHELD_FOR_UNVERIFIED_DEVICE:
return 'The sender only shares keys with verified sessions. Verify this session to read new messages.'
case DecryptionFailureCode.MEGOLM_KEY_WITHHELD:
return 'The sender’s session didn’t share the key to this message with you.'
case DecryptionFailureCode.OLM_UNKNOWN_MESSAGE_INDEX:
return 'This session only has the key to later messages from this sender.'
case DecryptionFailureCode.SENDER_IDENTITY_PREVIOUSLY_VERIFIED:
return 'Hidden: the sender’s identity changed since you verified them.'
case DecryptionFailureCode.UNSIGNED_SENDER_DEVICE:
return 'Hidden: sent from a session its owner hasn’t verified.'
case DecryptionFailureCode.MEGOLM_UNKNOWN_INBOUND_SESSION_ID:
return 'Waiting for the key to this message. It comes from the sender’s session, which may be offline.'
default:
return 'Can’t decrypt this message.'
}
}
/** Where to load a message's file or thumbnail from. */
export function mediaSource(m: MediaInfo, thumb = false): MediaSource | undefined {
if (thumb && m.thumbFile) return { file: m.thumbFile, mimetype: m.thumbMimetype ?? 'image/jpeg' }
if (thumb && m.thumbMxc) return m.thumbMxc
if (m.file) return { file: m.file, mimetype: m.mimetype }
return m.mxc
}
function mediaOf(content: RoomMessageEventContent): MediaInfo | undefined {
const msgtype = content.msgtype as string
if (msgtype !== MsgType.Image && msgtype !== MsgType.Video && msgtype !== MsgType.Audio && msgtype !== MsgType.File) {
return undefined
}
const c = content as RoomMessageEventContent & {
url?: string
file?: EncryptedFile
filename?: string
info?: {
mimetype?: string
size?: number
w?: number
h?: number
thumbnail_url?: string
thumbnail_file?: EncryptedFile
thumbnail_info?: { mimetype?: string }
}
}
return {
mxc: c.url ?? c.file?.url,
thumbMxc: c.info?.thumbnail_url ?? c.info?.thumbnail_file?.url,
file: c.file,
thumbFile: c.info?.thumbnail_file,
thumbMimetype: c.info?.thumbnail_info?.mimetype,
mimetype: c.info?.mimetype,
size: c.info?.size,
w: c.info?.w,
h: c.info?.h,
filename: c.filename || c.body || 'file',
}
}
/** Every event the view shows, oldest first: the live timeline then your pending ones. */
function displayEvents(room: Room): MatrixEvent[] {
return [...room.getLiveTimeline().getEvents(), ...room.getPendingEvents()]
}
export function buildTimeline(c: MatrixClient, room: Room): TimelineItem[] {
const me = c.getUserId() ?? ''
const cryptoRunning = Boolean(c.getCrypto())
// Messages after this were meant to be end-to-end encrypted.
const encryptionSince = room.currentState.getStateEvents(EventType.RoomEncryption, '')?.getTs() ?? Infinity
const events = displayEvents(room)
// An event is read when someone else's receipt is on it or on anything
// after it, or someone else has written after it.
const readByOthers = new Set<string>()
let seen = false
for (let i = events.length - 1; i >= 0; i--) {
const ev = events[i]!
const id = ev.getId()
if (!id || ev.status) continue
if (!seen) {
const sender = ev.getSender()
if (sender && sender !== me) seen = true
else if (room.getUsersReadUpTo(ev).some((u) => u !== me)) seen = true
}
if (seen) readByOthers.add(id)
}
const items: TimelineItem[] = []
let lastTs = 0
let prev: MessageItem | null = null
const calls = new CallTracker(room)
for (const ev of events) {
const type = ev.getType()
if (ev.isRelation('m.replace') || type === EventType.Reaction || type === EventType.RoomRedaction) continue
if (ev.status === EventStatus.CANCELLED) continue
const ts = ev.getTs()
if (!lastTs || !sameDay(lastTs, ts)) {
items.push({ kind: 'day', key: `day-${ts}`, ts })
prev = null
}
lastTs = ts
const key = ev.getTxnId() ?? ev.getId() ?? `${ts}-${items.length}`
const isMessage = type === EventType.RoomMessage || type === EventType.Sticker || type === EventType.RoomMessageEncrypted
if (!isMessage) {
if (!ev.isState()) {
const text = describeCall(ev, room)
if (text) {
items.push({ kind: 'notice', key, ts, text })
prev = null
}
continue
}
const text = type === EventType.GroupCallMemberPrefix ? calls.describe(ev, room) : describeState(ev, room)
if (text) {
items.push({ kind: 'notice', key, ts, text })
prev = null
}
continue
}
// A verification request in a direct message is a notice, not a message.
if (type === EventType.RoomMessage && ev.getContent().msgtype === 'm.key.verification.request') {
const to = String(ev.getContent().to ?? '')
items.push({ kind: 'notice', key, ts, text: `${nameOf(room, ev.getSender() ?? '')} started verifying ${nameOf(room, to)}` })
prev = null
continue
}
const sender = ev.getSender() ?? ''
const own = sender === me
const content = ev.getContent<RoomMessageEventContent>()
// Still ciphertext, or failed: the SDK then makes it an m.room.message
// with its own "Unable to decrypt" text, which is never shown.
const encrypted = type === EventType.RoomMessageEncrypted || ev.isDecryptionFailure()
const redacted = ev.isRedacted()
const status = own ? statusOf(ev) : undefined
const item: MessageItem = {
kind: 'message',
key,
eventId: ev.getId(),
sender,
senderName: nameOf(room, sender),
avatarMxc: room.getMember(sender)?.getMxcAvatarUrl() ?? undefined,
own,
ts,
msgtype: type === EventType.Sticker ? 'm.sticker' : String(content.msgtype ?? ''),
body: redacted ? '' : String(content.body ?? ''),
media: redacted || encrypted ? undefined : type === EventType.Sticker ? stickerMedia(ev) : mediaOf(content),
edited: Boolean(ev.replacingEventId()),
redacted,
encrypted,
utd: encrypted ? utdText(ev, cryptoRunning) : undefined,
e2ee: ev.isEncrypted(),
shield: cryptoRunning && !redacted ? shieldOf(c, ev, ts > encryptionSince) : undefined,
status: status === 'sent' && ev.getId() && readByOthers.has(ev.getId()!) ? 'read' : status,
first: true,
last: true,
event: ev,
}
if (prev && prev.sender === sender && ts - prev.ts < groupGapMs) {
item.first = false
prev.last = false
}
items.push(item)
prev = item
}
// Only the newest message of a run says "Sent" or "Read"; one that is
// still sending or failed always says so.
for (const it of items) {
if (it.kind === 'message' && it.own && !it.last && (it.status === 'sent' || it.status === 'read')) {
it.status = undefined
}
}
return items
}
function stickerMedia(ev: MatrixEvent): MediaInfo {
const c = ev.getContent() as { url?: string; body?: string; info?: { w?: number; h?: number; mimetype?: string } }
return { mxc: c.url, w: c.info?.w, h: c.info?.h, mimetype: c.info?.mimetype, filename: c.body ?? 'sticker' }
}
/** Whether there is older history to page back to. */
export function canPaginateBack(room: Room): boolean {
const tl = room.getLiveTimeline()
if (tl.getEvents().some((e) => e.getType() === EventType.RoomCreate)) return false
return tl.getPaginationToken(Direction.Backward) !== null
}
/** The newest event that came from the server, for read receipts. */
export function lastSyncedEvent(room: Room): MatrixEvent | undefined {
const events = room.getLiveTimeline().getEvents()
for (let i = events.length - 1; i >= 0; i--) {
const ev = events[i]!
if (ev.getId() && !ev.status && !ev.getId()!.startsWith('~')) return ev
}
return undefined
}