CI / Test (ubuntu-latest) (push) Canceled after 0s
CI / Test (windows-latest) (push) Canceled after 0s
CI / Storage tests on PostgreSQL (push) Canceled after 0s
CI / Complement (federation off) (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
CI / Smoke test the binary (push) Canceled after 0s
CI / Admin panel (push) Canceled after 0s
CI / Web client (push) Canceled after 0s
Every call in the web client now goes through MatrixRTC with a LiveKit
SFU, the model Element Call and Element X use; the legacy 1:1 m.call.*
calls are gone. In a room of two the other person is rung and can answer
or decline; in a bigger room a banner offers to join the call. Calls show
a grid of participant tiles with mute, camera, screen sharing and leave,
and media is end-to-end encrypted in encrypted rooms.
Server:
- MSC4140 delayed events (send/state with a delay, plus the
delayed_events list/restart/cancel/send API), stored and scheduled, so
a member whose client drops off is taken out of the call.
- POST /user/{userId}/openid/request_token, which lk-jwt-service uses to
hand out LiveKit tokens.
- org.matrix.msc4143.rtc_foci in /.well-known/matrix/client, configured
under client_api.rtc.
- Room presets let members send call membership state.
- Migration 0014 for both SQL dialects.
Deploy: LiveKit and lk-jwt-service in compose (calls.yml) and the helm
chart. Docs: docs/calls.md, configuration and deployment.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
442 lines
16 KiB
TypeScript
442 lines
16 KiB
TypeScript
import {
|
||
Direction,
|
||
EventStatus,
|
||
EventType,
|
||
MsgType,
|
||
type MatrixClient,
|
||
type MatrixEvent,
|
||
type Room,
|
||
} from 'matrix-js-sdk'
|
||
import { DecryptionFailureCode } from 'matrix-js-sdk/lib/crypto-api'
|
||
import type { RoomMessageEventContent } from 'matrix-js-sdk/lib/types'
|
||
import type { MessageStatus } from '@warp/ui'
|
||
import { sameDay } from '@/lib/format'
|
||
import type { EncryptedFile } from './attachments'
|
||
import type { MediaSource } from './media'
|
||
import { shieldOf, type Shield } from './trust'
|
||
|
||
// Turns a room's live timeline into what the timeline view draws: messages
|
||
// grouped by sender, day separators and one-line notices for state changes.
|
||
// The SDK keeps the events; this is a pure projection of them.
|
||
|
||
export interface MediaInfo {
|
||
mxc?: string
|
||
/** The thumbnail the sender uploaded, when there is one. */
|
||
thumbMxc?: string
|
||
/** Set in an encrypted room: the file is ciphertext, this decrypts it. */
|
||
file?: EncryptedFile
|
||
thumbFile?: EncryptedFile
|
||
thumbMimetype?: string
|
||
mimetype?: string
|
||
size?: number
|
||
w?: number
|
||
h?: number
|
||
filename: string
|
||
}
|
||
|
||
export interface MessageItem {
|
||
kind: 'message'
|
||
key: string
|
||
eventId?: string
|
||
sender: string
|
||
senderName: string
|
||
avatarMxc?: string
|
||
own: boolean
|
||
ts: number
|
||
msgtype: string
|
||
body: string
|
||
media?: MediaInfo
|
||
edited: boolean
|
||
redacted: boolean
|
||
/** Still ciphertext: being decrypted, or it can't be (see `utd`). */
|
||
encrypted: boolean
|
||
/** Why this session can't decrypt it. */
|
||
utd?: string
|
||
/** Sent end-to-end encrypted. */
|
||
e2ee: boolean
|
||
/** Why this message may not be from who it says, when that's in doubt. */
|
||
shield?: Shield
|
||
status?: MessageStatus
|
||
/** First of a run of messages from one sender: shows the name. */
|
||
first: boolean
|
||
/** Last of a run: shows the time and status. */
|
||
last: boolean
|
||
event: MatrixEvent
|
||
}
|
||
|
||
export interface NoticeItem {
|
||
kind: 'notice'
|
||
key: string
|
||
ts: number
|
||
text: string
|
||
}
|
||
|
||
export interface DayItem {
|
||
kind: 'day'
|
||
key: string
|
||
ts: number
|
||
}
|
||
|
||
export type TimelineItem = MessageItem | NoticeItem | DayItem
|
||
|
||
// Messages from one sender this close together form one group.
|
||
const groupGapMs = 5 * 60_000
|
||
|
||
function nameOf(room: Room, userId: string): string {
|
||
return room.getMember(userId)?.name ?? userId
|
||
}
|
||
|
||
function statusOf(ev: MatrixEvent): MessageStatus | undefined {
|
||
switch (ev.status) {
|
||
case EventStatus.NOT_SENT:
|
||
return 'failed'
|
||
case EventStatus.SENDING:
|
||
case EventStatus.QUEUED:
|
||
case EventStatus.ENCRYPTING:
|
||
return 'sending'
|
||
case EventStatus.CANCELLED:
|
||
return undefined
|
||
default:
|
||
return 'sent'
|
||
}
|
||
}
|
||
|
||
/** Whether an m.call.invite offers video. */
|
||
export function isVideoCallInvite(ev: MatrixEvent): boolean {
|
||
const sdp = ev.getContent().offer?.sdp
|
||
return typeof sdp === 'string' && sdp.includes('m=video')
|
||
}
|
||
|
||
/** A one-line description of a call event, or null for the signalling in between. */
|
||
function describeCall(ev: MatrixEvent, room: Room): string | null {
|
||
const sender = nameOf(room, ev.getSender() ?? '')
|
||
switch (ev.getType()) {
|
||
case EventType.CallInvite:
|
||
return `${sender} started a ${isVideoCallInvite(ev) ? 'video' : 'voice'} call`
|
||
case EventType.CallReject:
|
||
case EventType.RTCDecline:
|
||
return `${sender} declined the call`
|
||
case EventType.CallHangup:
|
||
switch (ev.getContent().reason) {
|
||
case 'invite_timeout':
|
||
return 'The call wasn’t answered'
|
||
case 'user_busy':
|
||
return `${sender} was busy`
|
||
case 'ice_failed':
|
||
return 'The call couldn’t connect'
|
||
default:
|
||
return `${sender} ended the call`
|
||
}
|
||
default:
|
||
return null
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Whether a call.member state event (MatrixRTC) says its device is in the
|
||
* call. Leaving is an empty event; the legacy form lists memberships.
|
||
*/
|
||
export function isActiveCallMember(content: Record<string, unknown>): boolean {
|
||
if (Object.keys(content).length === 0) return false
|
||
if (Array.isArray(content.memberships)) return content.memberships.length > 0
|
||
return true
|
||
}
|
||
|
||
/**
|
||
* Follows who is in a room's group call along the timeline, to say when
|
||
* it started and ended. Each device in the call has a call.member state
|
||
* event of its own; a member refreshing theirs says nothing.
|
||
*/
|
||
class CallTracker {
|
||
private readonly inCall = new Set<string>()
|
||
|
||
constructor(room: Room) {
|
||
const start = room.getLiveTimeline().getState(Direction.Backward)
|
||
for (const ev of start?.getStateEvents(EventType.GroupCallMemberPrefix) ?? []) {
|
||
if (isActiveCallMember(ev.getContent())) this.inCall.add(ev.getStateKey() ?? '')
|
||
}
|
||
}
|
||
|
||
describe(ev: MatrixEvent, room: Room): string | null {
|
||
const key = ev.getStateKey() ?? ''
|
||
const was = this.inCall.has(key)
|
||
const now = isActiveCallMember(ev.getContent())
|
||
if (was === now) return null
|
||
const sender = nameOf(room, ev.getSender() ?? '')
|
||
if (now) {
|
||
const first = this.inCall.size === 0
|
||
this.inCall.add(key)
|
||
return first ? `${sender} started a call` : `${sender} joined the call`
|
||
}
|
||
this.inCall.delete(key)
|
||
return this.inCall.size === 0 ? 'The call ended' : `${sender} left the call`
|
||
}
|
||
}
|
||
|
||
/** A one-line description of a state change, or null to leave it out. */
|
||
function describeState(ev: MatrixEvent, room: Room): string | null {
|
||
const sender = nameOf(room, ev.getSender() ?? '')
|
||
const content = ev.getContent()
|
||
const prev = ev.getPrevContent()
|
||
switch (ev.getType()) {
|
||
case EventType.RoomCreate:
|
||
return `${sender} created the room`
|
||
case EventType.RoomName:
|
||
return content.name ? `${sender} named the room ${content.name}` : `${sender} removed the room name`
|
||
case EventType.RoomTopic:
|
||
return content.topic ? `${sender} changed the topic to “${content.topic}”` : `${sender} removed the topic`
|
||
case EventType.RoomAvatar:
|
||
return `${sender} changed the room avatar`
|
||
case EventType.RoomEncryption:
|
||
return `${sender} turned on end-to-end encryption`
|
||
case EventType.RoomTombstone:
|
||
return `${sender} upgraded this room`
|
||
case EventType.RoomMember: {
|
||
const target = ev.getStateKey() ?? ''
|
||
const targetName = content.displayname ?? prev.displayname ?? target
|
||
const reason = content.reason ? `: ${content.reason}` : ''
|
||
switch (content.membership) {
|
||
case 'join':
|
||
if (prev.membership === 'join') {
|
||
if (content.displayname !== prev.displayname) {
|
||
if (!content.displayname) return `${prev.displayname ?? target} removed their display name`
|
||
if (!prev.displayname) return `${target} set their display name to ${content.displayname}`
|
||
return `${prev.displayname} changed their name to ${content.displayname}`
|
||
}
|
||
if (content.avatar_url !== prev.avatar_url) return `${targetName} changed their avatar`
|
||
return null
|
||
}
|
||
return `${targetName} joined`
|
||
case 'invite':
|
||
return `${sender} invited ${targetName}`
|
||
case 'leave':
|
||
if (target !== ev.getSender()) {
|
||
if (prev.membership === 'ban') return `${sender} unbanned ${targetName}`
|
||
if (prev.membership === 'invite') return `${sender} withdrew ${targetName}’s invitation`
|
||
return `${sender} removed ${targetName}${reason}`
|
||
}
|
||
return prev.membership === 'invite' ? `${targetName} declined the invitation` : `${targetName} left`
|
||
case 'ban':
|
||
return `${sender} banned ${targetName}${reason}`
|
||
case 'knock':
|
||
return `${targetName} asked to join`
|
||
}
|
||
return null
|
||
}
|
||
}
|
||
return null
|
||
}
|
||
|
||
/** What a message this session can't decrypt says about why, honestly. */
|
||
export function utdText(ev: MatrixEvent, cryptoRunning: boolean): string | undefined {
|
||
if (!ev.isDecryptionFailure()) return cryptoRunning ? undefined : 'End-to-end encryption didn’t start in this browser, so it can’t read this message.'
|
||
switch (ev.decryptionFailureReason) {
|
||
case DecryptionFailureCode.HISTORICAL_MESSAGE_NO_KEY_BACKUP:
|
||
case DecryptionFailureCode.HISTORICAL_MESSAGE_BACKUP_UNCONFIGURED:
|
||
return 'Sent before this session signed in. Verify this session or restore your key backup to read it.'
|
||
case DecryptionFailureCode.HISTORICAL_MESSAGE_WORKING_BACKUP:
|
||
return 'Sent before this session signed in, and its key isn’t in your key backup.'
|
||
case DecryptionFailureCode.HISTORICAL_MESSAGE_USER_NOT_JOINED:
|
||
return 'Sent before you joined this room, so you can’t read it.'
|
||
case DecryptionFailureCode.MEGOLM_KEY_WITHHELD_FOR_UNVERIFIED_DEVICE:
|
||
return 'The sender only shares keys with verified sessions. Verify this session to read new messages.'
|
||
case DecryptionFailureCode.MEGOLM_KEY_WITHHELD:
|
||
return 'The sender’s session didn’t share the key to this message with you.'
|
||
case DecryptionFailureCode.OLM_UNKNOWN_MESSAGE_INDEX:
|
||
return 'This session only has the key to later messages from this sender.'
|
||
case DecryptionFailureCode.SENDER_IDENTITY_PREVIOUSLY_VERIFIED:
|
||
return 'Hidden: the sender’s identity changed since you verified them.'
|
||
case DecryptionFailureCode.UNSIGNED_SENDER_DEVICE:
|
||
return 'Hidden: sent from a session its owner hasn’t verified.'
|
||
case DecryptionFailureCode.MEGOLM_UNKNOWN_INBOUND_SESSION_ID:
|
||
return 'Waiting for the key to this message. It comes from the sender’s session, which may be offline.'
|
||
default:
|
||
return 'Can’t decrypt this message.'
|
||
}
|
||
}
|
||
|
||
/** Where to load a message's file or thumbnail from. */
|
||
export function mediaSource(m: MediaInfo, thumb = false): MediaSource | undefined {
|
||
if (thumb && m.thumbFile) return { file: m.thumbFile, mimetype: m.thumbMimetype ?? 'image/jpeg' }
|
||
if (thumb && m.thumbMxc) return m.thumbMxc
|
||
if (m.file) return { file: m.file, mimetype: m.mimetype }
|
||
return m.mxc
|
||
}
|
||
|
||
function mediaOf(content: RoomMessageEventContent): MediaInfo | undefined {
|
||
const msgtype = content.msgtype as string
|
||
if (msgtype !== MsgType.Image && msgtype !== MsgType.Video && msgtype !== MsgType.Audio && msgtype !== MsgType.File) {
|
||
return undefined
|
||
}
|
||
const c = content as RoomMessageEventContent & {
|
||
url?: string
|
||
file?: EncryptedFile
|
||
filename?: string
|
||
info?: {
|
||
mimetype?: string
|
||
size?: number
|
||
w?: number
|
||
h?: number
|
||
thumbnail_url?: string
|
||
thumbnail_file?: EncryptedFile
|
||
thumbnail_info?: { mimetype?: string }
|
||
}
|
||
}
|
||
return {
|
||
mxc: c.url ?? c.file?.url,
|
||
thumbMxc: c.info?.thumbnail_url ?? c.info?.thumbnail_file?.url,
|
||
file: c.file,
|
||
thumbFile: c.info?.thumbnail_file,
|
||
thumbMimetype: c.info?.thumbnail_info?.mimetype,
|
||
mimetype: c.info?.mimetype,
|
||
size: c.info?.size,
|
||
w: c.info?.w,
|
||
h: c.info?.h,
|
||
filename: c.filename || c.body || 'file',
|
||
}
|
||
}
|
||
|
||
/** Every event the view shows, oldest first: the live timeline then your pending ones. */
|
||
function displayEvents(room: Room): MatrixEvent[] {
|
||
return [...room.getLiveTimeline().getEvents(), ...room.getPendingEvents()]
|
||
}
|
||
|
||
export function buildTimeline(c: MatrixClient, room: Room): TimelineItem[] {
|
||
const me = c.getUserId() ?? ''
|
||
const cryptoRunning = Boolean(c.getCrypto())
|
||
// Messages after this were meant to be end-to-end encrypted.
|
||
const encryptionSince = room.currentState.getStateEvents(EventType.RoomEncryption, '')?.getTs() ?? Infinity
|
||
const events = displayEvents(room)
|
||
|
||
// An event is read when someone else's receipt is on it or on anything
|
||
// after it, or someone else has written after it.
|
||
const readByOthers = new Set<string>()
|
||
let seen = false
|
||
for (let i = events.length - 1; i >= 0; i--) {
|
||
const ev = events[i]!
|
||
const id = ev.getId()
|
||
if (!id || ev.status) continue
|
||
if (!seen) {
|
||
const sender = ev.getSender()
|
||
if (sender && sender !== me) seen = true
|
||
else if (room.getUsersReadUpTo(ev).some((u) => u !== me)) seen = true
|
||
}
|
||
if (seen) readByOthers.add(id)
|
||
}
|
||
|
||
const items: TimelineItem[] = []
|
||
let lastTs = 0
|
||
let prev: MessageItem | null = null
|
||
const calls = new CallTracker(room)
|
||
|
||
for (const ev of events) {
|
||
const type = ev.getType()
|
||
if (ev.isRelation('m.replace') || type === EventType.Reaction || type === EventType.RoomRedaction) continue
|
||
if (ev.status === EventStatus.CANCELLED) continue
|
||
|
||
const ts = ev.getTs()
|
||
if (!lastTs || !sameDay(lastTs, ts)) {
|
||
items.push({ kind: 'day', key: `day-${ts}`, ts })
|
||
prev = null
|
||
}
|
||
lastTs = ts
|
||
|
||
const key = ev.getTxnId() ?? ev.getId() ?? `${ts}-${items.length}`
|
||
const isMessage = type === EventType.RoomMessage || type === EventType.Sticker || type === EventType.RoomMessageEncrypted
|
||
|
||
if (!isMessage) {
|
||
if (!ev.isState()) {
|
||
const text = describeCall(ev, room)
|
||
if (text) {
|
||
items.push({ kind: 'notice', key, ts, text })
|
||
prev = null
|
||
}
|
||
continue
|
||
}
|
||
const text = type === EventType.GroupCallMemberPrefix ? calls.describe(ev, room) : describeState(ev, room)
|
||
if (text) {
|
||
items.push({ kind: 'notice', key, ts, text })
|
||
prev = null
|
||
}
|
||
continue
|
||
}
|
||
|
||
// A verification request in a direct message is a notice, not a message.
|
||
if (type === EventType.RoomMessage && ev.getContent().msgtype === 'm.key.verification.request') {
|
||
const to = String(ev.getContent().to ?? '')
|
||
items.push({ kind: 'notice', key, ts, text: `${nameOf(room, ev.getSender() ?? '')} started verifying ${nameOf(room, to)}` })
|
||
prev = null
|
||
continue
|
||
}
|
||
|
||
const sender = ev.getSender() ?? ''
|
||
const own = sender === me
|
||
const content = ev.getContent<RoomMessageEventContent>()
|
||
// Still ciphertext, or failed: the SDK then makes it an m.room.message
|
||
// with its own "Unable to decrypt" text, which is never shown.
|
||
const encrypted = type === EventType.RoomMessageEncrypted || ev.isDecryptionFailure()
|
||
const redacted = ev.isRedacted()
|
||
const status = own ? statusOf(ev) : undefined
|
||
const item: MessageItem = {
|
||
kind: 'message',
|
||
key,
|
||
eventId: ev.getId(),
|
||
sender,
|
||
senderName: nameOf(room, sender),
|
||
avatarMxc: room.getMember(sender)?.getMxcAvatarUrl() ?? undefined,
|
||
own,
|
||
ts,
|
||
msgtype: type === EventType.Sticker ? 'm.sticker' : String(content.msgtype ?? ''),
|
||
body: redacted ? '' : String(content.body ?? ''),
|
||
media: redacted || encrypted ? undefined : type === EventType.Sticker ? stickerMedia(ev) : mediaOf(content),
|
||
edited: Boolean(ev.replacingEventId()),
|
||
redacted,
|
||
encrypted,
|
||
utd: encrypted ? utdText(ev, cryptoRunning) : undefined,
|
||
e2ee: ev.isEncrypted(),
|
||
shield: cryptoRunning && !redacted ? shieldOf(c, ev, ts > encryptionSince) : undefined,
|
||
status: status === 'sent' && ev.getId() && readByOthers.has(ev.getId()!) ? 'read' : status,
|
||
first: true,
|
||
last: true,
|
||
event: ev,
|
||
}
|
||
if (prev && prev.sender === sender && ts - prev.ts < groupGapMs) {
|
||
item.first = false
|
||
prev.last = false
|
||
}
|
||
items.push(item)
|
||
prev = item
|
||
}
|
||
|
||
// Only the newest message of a run says "Sent" or "Read"; one that is
|
||
// still sending or failed always says so.
|
||
for (const it of items) {
|
||
if (it.kind === 'message' && it.own && !it.last && (it.status === 'sent' || it.status === 'read')) {
|
||
it.status = undefined
|
||
}
|
||
}
|
||
return items
|
||
}
|
||
|
||
function stickerMedia(ev: MatrixEvent): MediaInfo {
|
||
const c = ev.getContent() as { url?: string; body?: string; info?: { w?: number; h?: number; mimetype?: string } }
|
||
return { mxc: c.url, w: c.info?.w, h: c.info?.h, mimetype: c.info?.mimetype, filename: c.body ?? 'sticker' }
|
||
}
|
||
|
||
/** Whether there is older history to page back to. */
|
||
export function canPaginateBack(room: Room): boolean {
|
||
const tl = room.getLiveTimeline()
|
||
if (tl.getEvents().some((e) => e.getType() === EventType.RoomCreate)) return false
|
||
return tl.getPaginationToken(Direction.Backward) !== null
|
||
}
|
||
|
||
/** The newest event that came from the server, for read receipts. */
|
||
export function lastSyncedEvent(room: Room): MatrixEvent | undefined {
|
||
const events = room.getLiveTimeline().getEvents()
|
||
for (let i = events.length - 1; i >= 0; i--) {
|
||
const ev = events[i]!
|
||
if (ev.getId() && !ev.status && !ev.getId()!.startsWith('~')) return ev
|
||
}
|
||
return undefined
|
||
}
|