From 76dfc98a86f4d3eff47e355f17953aad4d7ff013 Mon Sep 17 00:00:00 2001 From: tajniak81 <13187254+tajniak81@users.noreply.github.com> Date: Sun, 18 Oct 2026 19:33:29 +0200 Subject: [PATCH] Group voice and video calls with MatrixRTC and LiveKit Every call in the web client now goes through MatrixRTC with a LiveKit SFU, the model Element Call and Element X use; the legacy 1:1 m.call.* calls are gone. In a room of two the other person is rung and can answer or decline; in a bigger room a banner offers to join the call. Calls show a grid of participant tiles with mute, camera, screen sharing and leave, and media is end-to-end encrypted in encrypted rooms. Server: - MSC4140 delayed events (send/state with a delay, plus the delayed_events list/restart/cancel/send API), stored and scheduled, so a member whose client drops off is taken out of the call. - POST /user/{userId}/openid/request_token, which lk-jwt-service uses to hand out LiveKit tokens. - org.matrix.msc4143.rtc_foci in /.well-known/matrix/client, configured under client_api.rtc. - Room presets let members send call membership state. - Migration 0014 for both SQL dialects. Deploy: LiveKit and lk-jwt-service in compose (calls.yml) and the helm chart. Docs: docs/calls.md, configuration and deployment. Co-Authored-By: Claude Opus 5.5 --- apps/web/README.md | 24 +- apps/web/cmd/warp-web/main.go | 4 +- apps/web/internal/host/host.go | 29 + apps/web/internal/host/host_test.go | 32 + apps/web/package.json | 1 + apps/web/src/components/CallTile.vue | 58 ++ apps/web/src/components/CallView.vue | 286 +++--- apps/web/src/layouts/ClientShell.vue | 9 +- apps/web/src/matrix/client.ts | 3 + apps/web/src/matrix/timeline.ts | 45 +- apps/web/src/stores/call.ts | 880 ++++++++++++------ apps/web/src/stores/chat.ts | 5 +- apps/web/src/views/RoomView.vue | 42 +- apps/web/vite.config.ts | 2 + deploy/compose/calls.yml | 117 +++ deploy/compose/docker-compose.yml | 8 +- deploy/helm/warp/templates/_helpers.tpl | 29 + deploy/helm/warp/templates/deployment.yaml | 4 + deploy/helm/warp/templates/livekit-jwt.yaml | 107 +++ deploy/helm/warp/values.yaml | 47 + docs/README.md | 1 + docs/calls.md | 171 ++++ docs/configuration.md | 5 +- docs/deployment.md | 15 + pnpm-lock.yaml | 94 ++ server/internal/clientapi/clientapi.go | 25 + server/internal/clientapi/clientapi_test.go | 81 +- server/internal/clientapi/delayed.go | 259 ++++++ server/internal/clientapi/delayed_test.go | 300 ++++++ server/internal/clientapi/openid.go | 106 +++ server/internal/clientapi/openid_test.go | 61 ++ server/internal/clientapi/rooms.go | 21 + server/internal/clientapi/rtc.go | 51 + server/internal/clientapi/voip_test.go | 24 + server/internal/config/config.go | 48 + .../internal/delayedevents/delayedevents.go | 463 +++++++++ server/internal/httputil/errors.go | 5 + server/internal/roomserver/create.go | 8 + server/internal/setup/routes.go | 18 +- server/internal/setup/rtc_test.go | 87 ++ server/internal/setup/server.go | 32 + server/internal/storage/delayed.go | 110 +++ server/internal/storage/delayed_test.go | 147 +++ .../migrations/0014_delayed_events_openid.sql | 50 + .../migrations/0014_delayed_events_openid.sql | 50 + server/internal/storage/sqlstore/delayed.go | 273 ++++++ server/internal/storage/sqlstore/sqlstore.go | 10 + server/internal/storage/storage.go | 8 + warp.yaml.example | 26 +- 49 files changed, 3849 insertions(+), 432 deletions(-) create mode 100644 apps/web/src/components/CallTile.vue create mode 100644 deploy/compose/calls.yml create mode 100644 deploy/helm/warp/templates/livekit-jwt.yaml create mode 100644 docs/calls.md create mode 100644 server/internal/clientapi/delayed.go create mode 100644 server/internal/clientapi/delayed_test.go create mode 100644 server/internal/clientapi/openid.go create mode 100644 server/internal/clientapi/openid_test.go create mode 100644 server/internal/clientapi/rtc.go create mode 100644 server/internal/delayedevents/delayedevents.go create mode 100644 server/internal/setup/rtc_test.go create mode 100644 server/internal/storage/delayed.go create mode 100644 server/internal/storage/delayed_test.go create mode 100644 server/internal/storage/postgres/migrations/0014_delayed_events_openid.sql create mode 100644 server/internal/storage/sqlite/migrations/0014_delayed_events_openid.sql create mode 100644 server/internal/storage/sqlstore/delayed.go diff --git a/apps/web/README.md b/apps/web/README.md index be9e82a..1de27b6 100644 --- a/apps/web/README.md +++ b/apps/web/README.md @@ -63,12 +63,24 @@ that one origin. them asks you to acknowledge it; a reset since you *verified* them says their sessions are no longer trusted, with Verify again or Withdraw verification. -- Voice and video calls in a room with one other person (Matrix 1:1 VoIP, - WebRTC between the browsers): ringing, answer with or without video, - mute, camera on and off, screen sharing, and a panel that stays up while - you move between rooms and expands to fill the window. In an encrypted - room the call signalling is encrypted too. Calls outside one network - need a TURN server; see `client_api.turn` in the server configuration. +- Voice and video calls in any room, all through a LiveKit SFU (MatrixRTC, + the Element Call model; the old Matrix 1:1 VoIP is not used). Voice and + video call buttons in the room header. In a room with one other person + the call rings them: an incoming-call card with ring tone, answer, answer + without video and decline, and "Calling…" with a ringback tone for the + caller until they answer, decline or a minute passes. A call of two shows + the other person large and you in the corner; a bigger one is a grid + with a tile per person, their camera or screen, and a ring around + whoever is speaking. Mute, camera, screen sharing, leave, and a panel + that stays up while you move between rooms. A "Join call" banner while + one is going on in a room you're not in it. In an encrypted room the media + is end-to-end encrypted with per-participant keys. The room list says + when a room has a call going on, and the timeline says when one started + and ended. It needs `client_api.rtc` on the server and a LiveKit JWT + service; see [docs/calls.md](../../docs/calls.md). warp-web proxies + `/.well-known/matrix/client` for this, and `-call-origins` + (`WARP_WEB_CALL_ORIGINS`) lets the browser reach the JWT service and + LiveKit through the Content-Security-Policy. - Encryption setup in Settings: cross-signing keys, secret storage and a key backup in one step, behind a recovery key shown once. A new session verifies itself with the recovery key (or another session) and restores diff --git a/apps/web/cmd/warp-web/main.go b/apps/web/cmd/warp-web/main.go index 9e20ff9..e0e56f8 100644 --- a/apps/web/cmd/warp-web/main.go +++ b/apps/web/cmd/warp-web/main.go @@ -54,6 +54,8 @@ func run() error { "base URL of the homeserver, proxied for the client (WARP_WEB_HOMESERVER)") public := flag.String("public-homeserver", env("WARP_WEB_PUBLIC_HOMESERVER", ""), "public URL of the homeserver; when set, the browser calls it directly and nothing is proxied (WARP_WEB_PUBLIC_HOMESERVER)") + callOrigins := flag.String("call-origins", env("WARP_WEB_CALL_ORIGINS", ""), + "space-separated origins of the LiveKit JWT service and SFU the browser may connect to for group calls (WARP_WEB_CALL_ORIGINS)") logFormat := flag.String("log-format", env("WARP_WEB_LOG_FORMAT", "text"), "text or json (WARP_WEB_LOG_FORMAT)") showVersion := flag.Bool("version", false, "print the version and exit") flag.Parse() @@ -74,7 +76,7 @@ func run() error { } slog.SetDefault(logger) - opts := host.Options{Logger: logger, Version: version} + opts := host.Options{Logger: logger, Version: version, CallOrigins: strings.Fields(*callOrigins)} mode, target := "proxy", *homeserver if *public != "" { u, err := parseURL("public-homeserver", *public) diff --git a/apps/web/internal/host/host.go b/apps/web/internal/host/host.go index 82f4faf..ec1aba4 100644 --- a/apps/web/internal/host/host.go +++ b/apps/web/internal/host/host.go @@ -41,6 +41,11 @@ type Options struct { Logger *slog.Logger // Transport overrides the proxy's round tripper, for tests. Transport http.RoundTripper + // CallOrigins are origins outside the homeserver the browser may + // connect to for group calls: the LiveKit JWT service and the LiveKit + // SFU's WebSocket, e.g. https://livekit-jwt.example and + // wss://livekit.example. They are added to the CSP's connect-src. + CallOrigins []string } // proxiedPrefixes are the parts of the homeserver a Matrix client uses: the @@ -48,6 +53,10 @@ type Options struct { // are not reachable through warp-web. var proxiedPrefixes = []string{"/_matrix/client/", "/_matrix/media/"} +// wellKnownClientPath is proxied too: it is where the client finds the +// group call backend (org.matrix.msc4143.rtc_foci). +const wellKnownClientPath = "/.well-known/matrix/client" + // Config is the body of /config.json, which the client reads on start. type Config struct { // HomeserverURL is where the client sends Matrix requests; empty @@ -78,6 +87,22 @@ func New(opts Options) (http.Handler, error) { cfg.HomeserverURL = strings.TrimRight(u.String(), "/") connect += " " + u.Scheme + "://" + u.Host } + for _, raw := range opts.CallOrigins { + u, err := url.Parse(strings.TrimRight(raw, "/")) + if err != nil || u.Host == "" || u.Path != "" || u.RawQuery != "" || + (u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "ws" && u.Scheme != "wss") { + return nil, errors.New("host: a call origin must be an http(s) or ws(s) origin, got " + raw) + } + connect += " " + u.Scheme + "://" + u.Host + // LiveKit falls back from its WebSocket to an HTTP request on + // the same host to find out why a connection failed. + switch u.Scheme { + case "ws": + connect += " http://" + u.Host + case "wss": + connect += " https://" + u.Host + } + } body, err := json.Marshal(cfg) if err != nil { return nil, err @@ -167,6 +192,10 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok\n")) return + case p == wellKnownClientPath && h.proxy != nil: + h.proxy.ServeHTTP(w, r) + return + case strings.HasPrefix(p, "/_matrix/") || strings.HasPrefix(p, "/_warp/") || strings.HasPrefix(p, "/_synapse/"): // A path that cleans to something else (dot segments, doubled // slashes) is refused rather than normalised: what is allowed diff --git a/apps/web/internal/host/host_test.go b/apps/web/internal/host/host_test.go index fad83ab..211764e 100644 --- a/apps/web/internal/host/host_test.go +++ b/apps/web/internal/host/host_test.go @@ -304,3 +304,35 @@ func TestNewValidatesHomeserver(t *testing.T) { } } } + +func TestProxiesWellKnownClient(t *testing.T) { + up := newUpstream(t) + h := newHandler(t, up, true) + rec := do(h, http.MethodGet, "/.well-known/matrix/client", nil) + if rec.Code != 200 || len(up.hits) != 1 || up.hits[0].URL.Path != "/.well-known/matrix/client" { + t.Fatalf("status %d, hits %d", rec.Code, len(up.hits)) + } + // The server delegation document stays private. + if rec := do(h, http.MethodGet, "/.well-known/matrix/server", nil); rec.Code == 200 && len(up.hits) > 1 { + t.Errorf("/.well-known/matrix/server was proxied") + } +} + +func TestCallOriginsJoinConnectSrc(t *testing.T) { + up := newUpstream(t) + target, _ := url.Parse(up.srv.URL) + h, err := New(Options{Homeserver: target, Client: clientFS, + CallOrigins: []string{"https://livekit-jwt.warp.example/", "wss://livekit.warp.example"}}) + if err != nil { + t.Fatal(err) + } + rec := do(h, http.MethodGet, "/", nil) + want := "connect-src 'self' https://livekit-jwt.warp.example wss://livekit.warp.example https://livekit.warp.example;" + if csp := rec.Header().Get("Content-Security-Policy"); !strings.Contains(csp, want) { + t.Errorf("CSP %q", csp) + } + + if _, err := New(Options{Homeserver: target, CallOrigins: []string{"https://x.example/path"}}); err == nil { + t.Error("a call origin with a path was accepted") + } +} diff --git a/apps/web/package.json b/apps/web/package.json index 7e79c38..94487d9 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -13,6 +13,7 @@ "dependencies": { "@warp/ui": "workspace:*", "jsqr": "1.4.0", + "livekit-client": "2.22.3", "matrix-js-sdk": "^44.0.0", "pinia": "^3.0.3", "qrcode-generator": "1.4.4", diff --git a/apps/web/src/components/CallTile.vue b/apps/web/src/components/CallTile.vue new file mode 100644 index 0000000..23b06cf --- /dev/null +++ b/apps/web/src/components/CallTile.vue @@ -0,0 +1,58 @@ + + + diff --git a/apps/web/src/components/CallView.vue b/apps/web/src/components/CallView.vue index e145282..98c0f06 100644 --- a/apps/web/src/components/CallView.vue +++ b/apps/web/src/components/CallView.vue @@ -1,108 +1,74 @@