From 6aaf55433689fda38fbe7a1050a42710ccb27783 Mon Sep 17 00:00:00 2001 From: tajniak81 <13187254+tajniak81@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:14:20 +0200 Subject: [PATCH] Phase 5: federation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements PLAN.md Phase 5: the Server-Server API. Warp joins rooms on other servers and lets their users join its rooms, invites across servers with the invitee's server countersigning, rejects remote invites (locally if the inviter is unreachable), and exchanges events, typing, receipts and to-device messages with other homeservers — all behind the runtime switch, which now starts and stops real outbound traffic. The protocol lives in internal/federation with no knowledge of rooms: X-Matrix signing (fuzzed parser), the spec's discovery algorithm keeping the dial address and the TLS name apart, a signed client that refuses every request while the switch is off or the policy denies the destination, and a key ring that caches other servers' keys for at most a week, with optional notaries. The roomserver takes events from elsewhere through the spec's intake: signatures and hashes (a hash mismatch keeps the redacted form), the rules against the event's own auth events, against the state before it — with /get_missing_events for small gaps and /state_ids for wide ones — and against the current state, where a failure is a soft failure. Events held without their state are outliers: a column on roomserver_events, because the current state of a room joined over federation is almost all outliers, and excluded from every timeline query. Forks are resolved at the room's head. Membership across servers is make_join/send_join, make_leave/send_leave and countersigned invites, in both directions; m.room.server_acl is enforced on everything another server sends. History before a remote join is backfilled when /messages pages past it, stored at negative stream positions. internal/federationapi is the running component: the inbound endpoints, and an outbound queue that persists what it owes each destination, sends it in transactions of 50 PDUs and 100 EDUs, backs off from ten seconds to six hours, survives restarts, and drops events persisted while federation is off rather than delivering a backlog in one burst. Its Module is what the switch starts and stops. Also: a TLS listener for port 8448, trusted key servers, key validity and an outbound CA bundle in the config; per-destination health and backoff reset in the admin API; remote profiles, aliases and directories in the Client-Server API; a Complement federation profile. See docs/federation.md, including its known gaps (no partial-state joins, no key rotation). Co-Authored-By: Claude Opus 5.5 --- Makefile | 6 +- README.md | 73 +- docs/README.md | 1 + docs/admin-api.md | 37 + docs/configuration.md | 44 +- docs/development.md | 14 +- docs/federation.md | 380 ++++++ docs/rooms.md | 29 +- docs/sync.md | 17 +- server/internal/adminapi/adminapi.go | 8 + server/internal/adminapi/federation.go | 102 ++ server/internal/clientapi/clientapi.go | 6 + server/internal/clientapi/profile.go | 48 +- server/internal/clientapi/profile_test.go | 31 +- server/internal/clientapi/rooms.go | 8 + server/internal/clientapi/sync_test.go | 5 +- server/internal/config/config.go | 44 + server/internal/federation/client.go | 528 +++++++++ server/internal/federation/discovery.go | 322 +++++ server/internal/federation/discovery_test.go | 185 +++ server/internal/federation/gate.go | 12 +- server/internal/federation/keys.go | 463 ++++++++ server/internal/federation/keys_test.go | 259 ++++ server/internal/federation/requestauth.go | 68 ++ server/internal/federation/types.go | 151 +++ server/internal/federation/xmatrix.go | 239 ++++ server/internal/federation/xmatrix_test.go | 122 ++ server/internal/federationapi/edus.go | 98 ++ server/internal/federationapi/module.go | 96 ++ server/internal/federationapi/queue.go | 563 +++++++++ server/internal/federationapi/queue_test.go | 316 +++++ server/internal/federationapi/server.go | 661 +++++++++++ server/internal/matrix/event/event.go | 4 +- .../matrix/roomversion/roomversion.go | 4 +- server/internal/matrix/signing/key.go | 3 +- server/internal/roomserver/acl.go | 138 +++ .../internal/roomserver/acl_internal_test.go | 41 + server/internal/roomserver/backfill.go | 323 +++++ server/internal/roomserver/backfill_test.go | 222 ++++ server/internal/roomserver/directory.go | 15 +- server/internal/roomserver/federated.go | 689 +++++++++++ .../roomserver/federated_directory.go | 53 + .../roomserver/federated_membership.go | 787 ++++++++++++ server/internal/roomserver/federated_query.go | 373 ++++++ server/internal/roomserver/federated_test.go | 1054 +++++++++++++++++ server/internal/roomserver/input.go | 65 +- server/internal/roomserver/membership.go | 64 +- server/internal/roomserver/remote.go | 335 ++++++ server/internal/roomserver/roomserver.go | 48 + server/internal/roomserver/roomsync.go | 7 +- server/internal/roomserver/stream.go | 96 +- server/internal/roomserver/verify.go | 70 +- server/internal/roomserver/visibility.go | 17 +- server/internal/setup/federation.go | 77 ++ server/internal/setup/phase5_test.go | 441 +++++++ server/internal/setup/routes.go | 11 +- server/internal/setup/server.go | 256 +++- server/internal/storage/backfill_test.go | 205 ++++ server/internal/storage/federation.go | 123 ++ server/internal/storage/federation_test.go | 431 +++++++ .../internal/storage/pocketbase/pocketbase.go | 81 +- .../postgres/migrations/0005_federation.sql | 113 ++ .../postgres/migrations/0006_backfill.sql | 15 + server/internal/storage/rooms.go | 81 +- server/internal/storage/rooms_test.go | 2 +- .../sqlite/migrations/0005_federation.sql | 113 ++ .../sqlite/migrations/0006_backfill.sql | 15 + .../internal/storage/sqlstore/federation.go | 323 +++++ server/internal/storage/sqlstore/rooms.go | 263 +++- server/internal/storage/sqlstore/sequences.go | 11 +- server/internal/storage/sqlstore/sqlstore.go | 5 + server/internal/storage/sqlstore/timeline.go | 25 +- server/internal/storage/storage.go | 4 + server/internal/storage/sync.go | 24 +- server/internal/syncapi/context.go | 4 +- server/internal/syncapi/messages.go | 3 + server/internal/syncapi/outbound.go | 31 + server/internal/syncapi/presence.go | 2 +- server/internal/syncapi/receipts.go | 4 + server/internal/syncapi/syncapi.go | 4 + server/internal/syncapi/todevice.go | 25 +- server/internal/syncapi/token.go | 5 +- server/internal/syncapi/token_test.go | 9 +- server/internal/syncapi/typing.go | 3 + server/test/complement/Dockerfile | 14 +- server/test/complement/README.md | 27 +- server/test/complement/entrypoint.sh | 30 +- server/test/complement/run.sh | 56 +- warp.yaml.example | 22 + 89 files changed, 11940 insertions(+), 262 deletions(-) create mode 100644 docs/federation.md create mode 100644 server/internal/adminapi/federation.go create mode 100644 server/internal/federation/client.go create mode 100644 server/internal/federation/discovery.go create mode 100644 server/internal/federation/discovery_test.go create mode 100644 server/internal/federation/keys.go create mode 100644 server/internal/federation/keys_test.go create mode 100644 server/internal/federation/requestauth.go create mode 100644 server/internal/federation/types.go create mode 100644 server/internal/federation/xmatrix.go create mode 100644 server/internal/federation/xmatrix_test.go create mode 100644 server/internal/federationapi/edus.go create mode 100644 server/internal/federationapi/module.go create mode 100644 server/internal/federationapi/queue.go create mode 100644 server/internal/federationapi/queue_test.go create mode 100644 server/internal/federationapi/server.go create mode 100644 server/internal/roomserver/acl.go create mode 100644 server/internal/roomserver/acl_internal_test.go create mode 100644 server/internal/roomserver/backfill.go create mode 100644 server/internal/roomserver/backfill_test.go create mode 100644 server/internal/roomserver/federated.go create mode 100644 server/internal/roomserver/federated_directory.go create mode 100644 server/internal/roomserver/federated_membership.go create mode 100644 server/internal/roomserver/federated_query.go create mode 100644 server/internal/roomserver/federated_test.go create mode 100644 server/internal/roomserver/remote.go create mode 100644 server/internal/setup/federation.go create mode 100644 server/internal/setup/phase5_test.go create mode 100644 server/internal/storage/backfill_test.go create mode 100644 server/internal/storage/federation.go create mode 100644 server/internal/storage/federation_test.go create mode 100644 server/internal/storage/postgres/migrations/0005_federation.sql create mode 100644 server/internal/storage/postgres/migrations/0006_backfill.sql create mode 100644 server/internal/storage/sqlite/migrations/0005_federation.sql create mode 100644 server/internal/storage/sqlite/migrations/0006_backfill.sql create mode 100644 server/internal/storage/sqlstore/federation.go create mode 100644 server/internal/syncapi/outbound.go diff --git a/Makefile b/Makefile index 1b2b9a8..37c89ac 100644 --- a/Makefile +++ b/Makefile @@ -96,9 +96,13 @@ complement-image: ## Build the Complement homeserver image docker build -t $(COMPLEMENT_IMAGE) -f $(SERVER_DIR)/test/complement/Dockerfile . .PHONY: complement -complement: ## Run the Complement subset for the phases that are done +complement: ## Run the Complement C-S subset (federation off) bash $(SERVER_DIR)/test/complement/run.sh +.PHONY: complement-federation +complement-federation: ## Run the Complement federation subset + COMPLEMENT_PROFILE=federation bash $(SERVER_DIR)/test/complement/run.sh + .PHONY: docker docker: ## Build the container image docker build -t warp:$(VERSION) . diff --git a/README.md b/README.md index 27e3424..2847701 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,24 @@ what exists today. ## Status +**Phase 5 — federation. Complete.** Warp talks to other homeservers. It +joins rooms on other servers and lets their users join its rooms, invites +across servers with the invitee's server countersigning, rejects remote +invites (locally, if the inviter is gone), and exchanges events, typing, +read receipts and to-device messages with them. Events from elsewhere go +through the spec's full intake — signatures and hashes, the rules against +their own auth events, against the state before them and against the +current state, with missing history fetched and soft failure for what +fails only at the end — and forked rooms are resolved back together. The +outbound queue persists what it owes each server, retries with backoff, +and survives restarts. Server discovery, key fetching with an optional +notary, X-Matrix request signing, server ACLs, and a TLS listener for port +8448. The switch is real now: turning federation off stops the queue and +refuses every outbound request, turning it back on resumes from where it +stopped, and the admin API shows each server's queue and backoff. Known +gaps — no history backfill from other servers, no partial-state joins, no +key rotation — are listed in [docs/federation.md](docs/federation.md). + **Phase 4 — sync. Complete. Milestone M1.** Warp is now a homeserver a client can be used against. `/sync` is real: stream tokens across five independent streams, long polling that wakes the moment something concerns @@ -50,18 +68,18 @@ Client-Server API: | `GET`, `POST /login` · `POST /logout`, `/logout/all` · `POST /refresh` | sessions, with `m.login.password` and refresh tokens | | `GET /account/whoami` · `GET /capabilities` | who am I, and what this build can do | | `GET /devices`, `GET`/`PUT`/`DELETE /devices/{id}` · `POST /delete_devices` | device management; deletion re-asks for the password | -| `GET /profile/{user}` and its `displayname` / `avatar_url` sub-resources | profiles — reads are public, writes are your own | +| `GET /profile/{user}` and its `displayname` / `avatar_url` sub-resources | profiles — reads are public, remote ones asked of their server, writes are your own | | `GET`/`PUT /user/{user}/account_data/{type}` and the per-room variant | account data, stored and returned byte for byte | | `POST /createRoom` | room versions 3 to 12, presets, `initial_state`, invites, aliases | -| `POST /join/{roomIdOrAlias}` · `POST /rooms/{id}/join`, `/leave` | joining by ID or alias, and leaving or rejecting an invite | -| `POST /rooms/{id}/invite`, `/kick`, `/ban`, `/unban` | membership, decided by the room's power levels | +| `POST /join/{roomIdOrAlias}` · `POST /rooms/{id}/join`, `/leave` | joining by ID or alias — on another server too, with federation on — and leaving or rejecting an invite | +| `POST /rooms/{id}/invite`, `/kick`, `/ban`, `/unban` | membership, decided by the room's power levels; invites reach users of other servers | | `PUT /rooms/{id}/send/{type}/{txn}` | messages, idempotent per transaction ID | | `GET`/`PUT /rooms/{id}/state[/{type}[/{key}]]` | room state, read whole or one piece at a time | | `PUT /rooms/{id}/redact/{event}/{txn}` | redactions, applied only when they take effect | | `GET /rooms/{id}/event/{event}` · `/members` (with `?at=`) · `/joined_members` · `/aliases` | reading a room | | `GET /joined_rooms` | the rooms you are in | -| `GET`/`PUT`/`DELETE /directory/room/{alias}` | the alias directory | -| `GET`/`POST /publicRooms` · `GET`/`PUT /directory/list/room/{id}` | the public room directory | +| `GET`/`PUT`/`DELETE /directory/room/{alias}` | the alias directory; remote aliases are asked of their server | +| `GET`/`POST /publicRooms` · `GET`/`PUT /directory/list/room/{id}` | the public room directory, ours or (`?server=`) another server's | | `GET /sync` | the event stream: long-polled, incremental, filtered, with lazy-loaded members | | `GET /rooms/{id}/messages` · `/context/{event}` | paging a room's history, and one event with what surrounds it | | `POST /user/{user}/filter` · `GET /user/{user}/filter/{id}` | filters, stored and returned byte for byte | @@ -71,12 +89,14 @@ Client-Server API: | `GET`/`PUT /presence/{user}/status` | presence — **only while `sync.presence` is on**, and it is off by default | | `GET /_matrix/client/versions` · `/.well-known/matrix/client` | discovery | | `/.well-known/matrix/server` | server discovery — **only while federation is on** | +| `/_matrix/federation/*` · `/_matrix/key/*` | the Server-Server API — **only while federation is on**; see [docs/federation.md](docs/federation.md) | | `GET /healthz` · `GET /metrics` | liveness with a database ping, and Prometheus | | anything else under `/_matrix/` | `404 M_UNRECOGNIZED`, as the spec requires | Admin API under `/_warp/admin/v1`: the federation and registration -switches, user listing and creation, the room list and detail, purging a -room, removing a user from one, and the dashboard counters. See +switches, per-server federation health, user listing and creation, the +room list and detail, purging a room, removing a user from one, and the +dashboard counters. See [docs/admin-api.md](docs/admin-api.md). Also in place: argon2id passwords, hashed access and refresh tokens, @@ -85,11 +105,13 @@ logging, graceful shutdown, the storage interfaces with a working SQLite and PostgreSQL driver (and a PocketBase stub), the embedded migration runner and the runtime settings service. -Complement, the official integration suite, is wired up for the -federation-off profile: `make complement` runs the account, room and stream -tests — `TestRegistration`, `TestLogin`, `TestDeviceManagement`, the Phase 3 -room tests and the Phase 4 `TestSync*`, `TestMessages*`, `TestContext*`, -`TestFilter*`, `TestTyping*`, `TestReceipt*` and `TestToDevice*`. +Complement, the official integration suite, is wired up for both +profiles. `make complement` runs the account, room and stream tests with +federation off — `TestRegistration`, `TestLogin`, `TestDeviceManagement`, +the Phase 3 room tests and the Phase 4 `TestSync*`, `TestMessages*`, +`TestContext*`, `TestFilter*`, `TestTyping*`, `TestReceipt*` and +`TestToDevice*`. `make complement-federation` runs the Phase 5 +Server-Server tests with federation on, over TLS with Complement's CA. ## Quick start @@ -161,17 +183,26 @@ server has ever signed. ## Federation switch Federation is off in a fresh install. While it is off, the federation -surface does not answer at all; `/.well-known/matrix/server` returns -`404 M_UNRECOGNIZED`, exactly as it would on a server that has no -federation API. +surface does not answer at all: `/_matrix/federation/*`, `/_matrix/key/*` +and `/.well-known/matrix/server` return `404 M_UNRECOGNIZED`, exactly as +they would on a server that has no federation API, and nothing outbound +runs. The switch is stored in the `server_settings` table and cached in memory, -so flipping it takes effect on the next request without a restart. In -Phase 0 the only thing behind the gate is the delegation document; the -outbound queue, key fetcher and remote media fetch join it in Phase 5. Set +so flipping it — `PUT /_warp/admin/v1/federation {"enabled": true}` — +takes effect on the next request without a restart. Switching it on +starts the outbound queue, which picks up whatever it still owed other +servers; switching it off stops the queue mid-flight and makes the +federation client refuse every request before it opens a connection. Set `federation.allow_runtime_toggle: false` to pin the switch to the config file. +To federate, other servers have to be able to reach this one: either set +`global.well_known.server_address` to a reverse proxy that terminates TLS, +or let Warp terminate it with `listen.federation_address: ":8448"` and a +certificate. See [docs/configuration.md](docs/configuration.md) and +[docs/federation.md](docs/federation.md). + ## Development ```sh @@ -179,7 +210,8 @@ make test # unit tests (SQLite) make lint # golangci-lint make check # vet + test, what CI runs make cover # coverage.html -make complement # the Complement subset for the phases that are done (needs Docker) +make complement # the Complement C-S subset, federation off (needs Docker) +make complement-federation # the Complement federation subset # the storage tests against PostgreSQL as well WARP_TEST_POSTGRES_DSN='postgres://warp:warp@localhost/warp_test?sslmode=disable' make test-postgres @@ -198,7 +230,8 @@ server/ the homeserver (Go module) logging/ metrics/ slog and Prometheus setup httputil/ Matrix error envelope, CORS, router, middleware ratelimit/ per-endpoint token buckets - federation/ the federation gate and allow/deny policy + federation/ the Server-Server protocol: gate, X-Matrix, discovery, client, keys + federationapi/ federation endpoints, the outbound queue, the switchable Module (docs/federation.md) settings/ runtime settings service (the switches) userapi/ accounts, passwords, devices, tokens, account data roomserver/ rooms: the input pipeline (docs/rooms.md) diff --git a/docs/README.md b/docs/README.md index eafa7cc..d372734 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,6 +7,7 @@ | [event-core.md](event-core.md) | `internal/matrix`: canonical JSON, events, auth rules, state resolution | | [rooms.md](rooms.md) | `internal/roomserver`: the input pipeline, room storage, known gaps | | [sync.md](sync.md) | `internal/syncapi`: stream tokens, long polling, filters, the ephemeral streams | +| [federation.md](federation.md) | the Server-Server API: discovery, keys, taking in events, joins across servers, the outbound queue, the switch | | [admin-api.md](admin-api.md) | `/_warp/admin/v1`: the operator API behind the admin panel | The design and roadmap are in [PLAN.md](../PLAN.md) at the repository root. diff --git a/docs/admin-api.md b/docs/admin-api.md index cfa51b0..d35cafa 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -57,6 +57,43 @@ than offer something that will bounce. Editing the allow and deny lists is Phase 9. They are reported now because the dashboard has to show which servers a running Warp will talk to. +### `GET /federation/destinations` + +Delivery health per remote server: the panel's federation status view. +Every server this one has queued something for, or has failed to reach, +with how much it is still owed. + +```json +{ + "running": true, + "destinations": [ + { + "server_name": "matrix.org", + "failures": 0, + "retry_at_ms": 0, + "last_success_ms": 1760600000000, + "last_failure_ms": 0, + "queued_pdus": 0, + "queued_edus": 0 + } + ] +} +``` + +`running` is whether the outbound queue is up — `false` while federation is +switched off. `failures` counts consecutive failed sends; while it is above +zero the server is being backed off, and `retry_at_ms` is when the next +attempt is due. The backoff doubles from ten seconds to six hours. + +A server started without a signing key has no federation at all, and both +destination endpoints answer `404`. + +### `POST /federation/destinations/{serverName}/reset` + +Forget a server's backoff and try it now — for an operator who knows the +server is back and does not want to wait out a six-hour retry. Answers +`{}`. + ### `GET /registration`, `PUT /registration` The same shape, without the lists, for open registration. diff --git a/docs/configuration.md b/docs/configuration.md index d0e9556..113e05f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -33,6 +33,9 @@ List-valued variables are comma-separated. A single `-` clears a list: | `listen.idle_timeout` | `WARP_IDLE_TIMEOUT` | `120s` | | | `listen.max_header_bytes` | `WARP_MAX_HEADER_BYTES` | `1048576` | | | `listen.shutdown_timeout` | `WARP_SHUTDOWN_TIMEOUT` | `30s` | Grace period for in-flight requests. | +| `listen.federation_address` | `WARP_LISTEN_FEDERATION_ADDRESS` | empty | A second listener serving the same API over TLS, usually `:8448`. Empty = none; see *Federation over TLS*. | +| `listen.tls_cert_file` | `WARP_TLS_CERT_FILE` | empty | PEM certificate chain for the federation listener. Required with it. | +| `listen.tls_key_file` | `WARP_TLS_KEY_FILE` | empty | PEM key for the federation listener. Required with it. | | `database.driver` | `WARP_DB_DRIVER` | `sqlite` | `sqlite`, `postgres`, `pocketbase` (not implemented). | | `database.connection_string` | `WARP_DB_CONNECTION_STRING` | SQLite file in the working directory | | | `database.max_open_conns` | `WARP_DB_MAX_OPEN_CONNS` | `10` | Ignored by SQLite: writes are serialised onto one connection. | @@ -57,6 +60,9 @@ List-valued variables are comma-separated. A single `-` clears a list: | `federation.allowlist` | `WARP_FEDERATION_ALLOWLIST` | `[]` | Non-empty = federate only with these. | | `federation.denylist` | `WARP_FEDERATION_DENYLIST` | `[]` | Always wins over the allowlist. | | `federation.allow_runtime_toggle` | `WARP_FEDERATION_ALLOW_RUNTIME_TOGGLE` | `true` | See below. | +| `federation.trusted_key_servers` | `WARP_FEDERATION_TRUSTED_KEY_SERVERS` | `[]` | Notaries asked for a server's key when it cannot be reached itself, e.g. `matrix.org`. Empty = only ever ask the server. | +| `federation.key_validity_period` | `WARP_FEDERATION_KEY_VALIDITY_PERIOD` | `24h` | How long other servers may cache this server's key. At most `168h`. | +| `federation.tls_ca_file` | `WARP_FEDERATION_TLS_CA_FILE` | empty | Extra CA certificates (PEM) for outbound federation, on top of the system roots. | | `registration.enabled` | `WARP_REGISTRATION_ENABLED` | `false` | | | `registration.allow_runtime_toggle` | `WARP_REGISTRATION_ALLOW_RUNTIME_TOGGLE` | `true` | | | `registration.tokens` | `WARP_REGISTRATION_TOKENS` | `[]` | Non-empty = sign-up also needs one of these, as the `m.login.registration_token` stage. | @@ -145,13 +151,41 @@ Everything else is read once at startup. While `federation.enabled` is false: -- `/.well-known/matrix/server` answers `404 M_UNRECOGNIZED`, and so will - `/_matrix/federation/*` and `/_matrix/key/*` once they exist. -- Nothing outbound runs — in Phase 0 there is nothing outbound yet; from - Phase 5 the transaction queue, key fetcher and remote media fetch stay - stopped. +- `/_matrix/federation/*`, `/_matrix/key/*` and `/.well-known/matrix/server` + answer `404 M_UNRECOGNIZED`, exactly as on a server with no federation + API. The TLS listener on `listen.federation_address`, if there is one, + stays up and answers the same way. +- Nothing outbound runs. The transaction queue is stopped, and the + federation client refuses every request — key fetches, joins, profile + and alias lookups — before it opens a connection, so a component that + somehow missed the switch still cannot reach another server. +- Events persisted while it is off are not queued for other servers. A + remote server that cares about the room fetches what it missed the next + time it hears from this one. What was already queued when the switch + went off stays queued, and goes out when it comes back on. - Client-Server endpoints that would need a remote server answer `M_FORBIDDEN` with a message saying federation is disabled. - Events are still signed with the server key, and room and event IDs are unchanged. A room created while federation is off federates normally once it is switched on. + +## Federation over TLS + +Other servers reach this one at whatever its server name resolves to: the +address in `/.well-known/matrix/server`, an SRV record, or the name itself +on port 8448. Two deployment shapes: + +- **Behind a reverse proxy** that terminates TLS — the usual one. Set + `global.well_known.server_address` to the proxy's `host:443` and leave + `listen.federation_address` empty: the proxy forwards + `/_matrix/federation` and `/_matrix/key` to `listen.address` like + everything else. +- **Warp terminates TLS itself**: set `listen.federation_address` to + `:8448` with `listen.tls_cert_file` and `listen.tls_key_file`. The + certificate must be valid for the server name (or for the delegated + name, if the well-known document delegates). + +Outbound federation verifies certificates against the system roots, plus +`federation.tls_ca_file` when it is set — for a private federation whose +servers share an internal CA. There is no switch to turn verification +off. diff --git a/docs/development.md b/docs/development.md index 07916bd..6d73e9a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -147,15 +147,17 @@ far along the server is — PLAN.md section 9 treats it as the primary progress metric. ```sh -make complement # the current target set +make complement # the C-S set, federation off +make complement-federation # the federation set server/test/complement/run.sh TestLogin # one test COMPLEMENT_DEBUG=1 make complement # with the container logs ``` It needs Docker. The script builds the image, clones Complement into `.complement/` (ignored by git) and runs the subset for the phases that are -done — the account, room and stream tests, with federation off. Extend -`COMPLEMENT_TESTS` in +done. `COMPLEMENT_PROFILE` picks the set: `csapi` (the default) runs the +account, room and stream tests with federation off; `federation` runs the +Server-Server tests with it on. Extend the sets in [run.sh](../server/test/complement/run.sh) as each phase lands, rather than switching to the whole suite: a pass count that drops because a later phase is missing is noise. @@ -165,9 +167,9 @@ and the deployment is entirely `WARP_*` environment variables — see [entrypoint.sh](../server/test/complement/entrypoint.sh). Two things there are worth knowing: rate limiting is off, because the whole suite comes from one address and would otherwise spend its time reading `M_LIMIT_EXCEEDED`; -and port 8448 is exposed but nothing listens on it yet. The federation -profile needs a TLS listener signed by the CA that Complement mounts at -`/complement/ca`, which arrives with federation in Phase 5. +and port 8448 is served over TLS with a certificate the entrypoint has the +CA Complement mounts at `/complement/ca` sign, the same CA Warp is told to +trust for its outbound connections. ## Creating the first account diff --git a/docs/federation.md b/docs/federation.md new file mode 100644 index 0000000..c7d4006 --- /dev/null +++ b/docs/federation.md @@ -0,0 +1,380 @@ +# Federation + +PLAN.md Phase 5: the Server-Server API. Warp can now join rooms on other +servers, have their users join its rooms, exchange events and typing, +receipts and to-device messages with them, and do all of it behind a switch +that turns it off at runtime — cleanly, with nothing left talking to +another server — and back on again, picking up where it left off. + +Federation is still off in a fresh install. Nothing here changes what a +private homeserver does. + +## Where things live + +| Package | What | +|---|---| +| `internal/federation` | The protocol, with no knowledge of rooms: X-Matrix request signing, server discovery, the signed HTTP client, the key ring, the gate and the allow/deny policy. | +| `internal/roomserver` | Rooms across servers: taking in events from elsewhere (`federated.go`), joining, leaving and inviting across servers in both directions (`federated_membership.go`), answering other servers' reads (`federated_query.go`), remote aliases and directories, server ACLs (`acl.go`). | +| `internal/federationapi` | Federation as a running component: the inbound endpoints, the outbound queue, the EDU adapter and the `Module` the switch starts and stops. | + +The split follows the dependencies. The roomserver uses the protocol — it +fetches events and asks for joins — and the component uses the roomserver, +so the component cannot live in the protocol package. PLAN.md calls it +`federation.Module`; it is `federationapi.Module`, for that reason. + +## The switch + +Three things answer to `federation.enabled`, and each reads it at the +moment it matters rather than being told: + +- **The gate.** Every route under `/_matrix/federation`, `/_matrix/key` + and `/.well-known/matrix/server` is wrapped by it, and answers + `404 M_UNRECOGNIZED` while the switch is off — a server with federation + off looks exactly like a server without a federation API. +- **The client.** Every outbound request checks the switch, the allow and + deny lists, and that it is not addressed to this server, before it + resolves anything or opens a connection. A component that somehow missed + the toggle still cannot reach another server. +- **The module.** The one thing that talks to other servers on its own + initiative is the outbound queue, and the module starts and stops it. + The settings service calls the module when the switch moves; stopping + cancels in-flight sends and waits for them, and leaves whatever was not + sent in the database. + +Events persisted while the switch is off are not queued. Holding them would +turn switching federation back on after a month into a month of events +delivered in one burst; a remote server that cares about a room fetches +what it missed with `/get_missing_events` the next time it hears from this +one. What was queued before the switch went off goes out when it comes +back on. + +## Finding other servers + +`federation.Resolver` implements the spec's discovery algorithm, in its +order: an IP literal or an explicit port is used as is; otherwise +`/.well-known/matrix/server` may delegate to another name, which is then +resolved the same way; then the `_matrix-fed._tcp` SRV record, then the +deprecated `_matrix._tcp`, then port 8448. + +The result has two halves and the client keeps them apart. The address to +dial can be an SRV target; the name the TLS certificate must be valid for — +also the `Host` header — is never the SRV target, only the server name or +the name the well-known document delegated to. The client carries the +dial addresses down to its dialer in the request context and builds the URL +from the other half, so Go's TLS verification checks the right name +without any custom verification code. + +Well-known answers are cached for their `Cache-Control` max-age, between +the spec's bounds (24 hours by default, 48 at most); failures for an hour, +so that a server with no document is not asked for one on every request. + +## Keys + +`GET /_matrix/key/v2/server` serves this server's key, valid for +`federation.key_validity_period` (24 hours by default). + +Everybody else's keys go through the key ring. A key response is accepted +only if it is about the server asked for and every signature the server +made on it is by a key it lists and verifies — a response citing a key it +does not publish is broken or forged. Keys are stored in +`federation_server_keys` and cached in memory, and are believed for as long +as the server said, but never for more than a week: the spec's cap. + +When a server's own key endpoint cannot be reached, the notaries in +`federation.trusted_key_servers` are asked, in order; a notary's answer +must carry the notary's signature as well. The default is no notaries: +keys come from the server that owns them or not at all. + +Room versions 5 and later require a key to have been valid when the event +was sent. A cached key that is not known to cover an event's timestamp is +fetched again before the event is refused — the server may have extended +it — and a fetch that fails is not retried for a minute, so that a server +whose key endpoint is down is not asked on every event it sends. + +As a notary, Warp answers for its own keys only. It keeps other servers' +keys, not the signed documents they came in, and a notary has to hand out +the documents. + +## Authenticating requests + +Every federation endpoint but the key and version endpoints checks the +`X-Matrix` Authorization header: the signature covers the method, the +request target exactly as it appeared on the request line, the origin, the +destination and the JSON body. The origin is then checked against the +allow and deny lists, and — for anything about a room — against the room's +server ACL. + +The parser is fuzzed (`FuzzParseXMatrix`): a header that parses must +survive being written back out and parsed again unchanged. + +## Taking in events from other servers + +An event created here is built on the room's current state and can only +fail the authorisation rules. An event from somewhere else arrives with +none of that settled. The roomserver follows the spec's procedure, in +`federated.go`: + +1. **It is what it says it is.** It parses under the room's version, it is + signed by its sender's server, and its content matches its hash. A hash + mismatch with a good signature means somebody redacted it on the way; + the redacted form is kept rather than the event rejected. +2. **It passes the rules against its own auth events.** Missing auth + events are fetched with `/event_auth` and checked themselves, oldest + first. +3. **It passes them against the state before it** — the state after each + of its prev_events, resolved together. A prev_event this server does not + have is asked for with `/get_missing_events` (ten events, between this + server's forward extremities and the new event). A gap wider than that + is closed by asking for the state at the missing event with + `/state_ids`, fetching whatever of it is missing — one at a time with + `/event`, or all at once with `/state` past fifty — and checking all of + it. +4. **It passes them against the current state.** An event that fails only + here is *soft-failed*: kept, because other events may cite it, but never + shown to a client and never a forward extremity. This is what stops a + banned user's server from sending events built on the state before the + ban. + +An event that fails 1 to 3 is rejected and not stored. The `/send` +response says why, per event. + +### Outliers + +Some events have to be held without the room's state at them: the state +and auth chain a join is handed, auth events fetched to judge something +else, the state fetched across a gap, an invite to a room nobody here is +in, a soft-failed event. These are *outliers* — a column on +`roomserver_events` rather than a table of their own, because every read +of a room's current state joins the events table, and the current state of +a room joined over federation is almost all outliers. + +An outlier can be cited and read by ID. It has no state snapshot, is never +a forward extremity, and never appears in `/sync`, `/messages` or +`/context` — the timeline queries filter it out. An outlier that arrives +again as an ordinary event stays an outlier: this server used it as state +without its history, and putting the room's head on it now could put the +head behind events that are already here. + +### The room's head + +A Phase 3 room had one forward extremity, because every event was a child +of the one before. Federation forks rooms: two servers send at the same +moment, each without the other's event. The new extremities are the old +ones the event does not descend from, plus the event. + +The current state then has to be worked out again. In the common case — +the event follows the room's single extremity — it is one tuple more than +before. Otherwise it is resolved across the new extremities and written +whole. The next event created here cites every extremity, and the fork +closes; `TestFederatedJoinAndConversation` checks that both servers end up +with one extremity and the same state. + +### History: backfill + +A room joined over federation starts, on this server, at the join: the +join is handed the room's state, not the events that led to it. When a +client pages `/messages` backwards and the page comes back short at the +bottom of what this server holds, the roomserver asks the servers in the +room for what came before it (`GET /backfill`) before answering, in +`backfill.go`: + +- **Where to start.** The prev_events of the room's oldest timeline events + (the oldest fifty) that are not themselves in the timeline. A room whose + oldest event is its create event has nothing to ask for. +- **Whom to ask.** The servers of the room's members, joined ones first, + in turn, until one sends something that passes. A page asks at most + three times before it is returned short. +- **What is checked.** Steps 1 to 3 of the intake above. The state before + each event comes from the events before it in the same batch, or — + for the oldest ones — from `/state_ids` at its prev_events. Step 4 does + not apply: soft failure refuses events built on a state the room has + moved past, which history always is. An event already held as an + outlier is promoted into the timeline. +- **Where it goes.** Below everything. The stream ordering counter only + goes up, so backfilled events are numbered by a second counter that runs + downwards from -1 (`roomserver_sequences.backfill_ordering`). Each batch + sorts below the one before it, and nothing about the room's head, its + current state or its stream top changes: a live `/sync` reads "after my + token" and never reaches below zero, so history reaches clients only + through `/messages`, `/context` and the state at a point in it. Zero is + never a position, so it can go on meaning "unbounded" in every window; + the position just below the event at 1 is -1 (`storage.PositionBefore`). + +Backfill is only asked for when the user may see before their join — a +room whose history visibility is `joined` or `invited` has nothing to +show them there — and only by `/messages`, never by `/sync` or `/context`, +which do not wait on another server. With federation off, or with nobody +reachable, the page is what this server holds. + +## Membership across servers + +A server can only sign as itself, and a room's state lives with the +servers in it, so each membership change that crosses a server boundary is +a short protocol: + +- **Joining a room this server is not in** — not held at all, held only as + an invite, or held from before every local user left — asks a resident + server for a template (`make_join`), fills it in, signs it, and hands it + back (`send_join`, v2). The answer is the room's state and auth chain, + every event of which is verified and checked against its own auth events + before any of it is believed. Candidates are tried in order: the + client's `server_name` hints, the server in the room ID (before version + 12), the servers of the room's members as last known, and whoever sent + the invite. A restricted join comes back countersigned by the server that + authorised it. +- **Leaving a room this server is not in** — rejecting an invite, in + practice — is the same with `make_leave` and `send_leave`. When no server + in the room can be reached the invite is still rejected locally: a leave + is built and stored here and nowhere else. An invite a user cannot get + rid of because its server is gone is the worse failure. +- **Inviting a user of another server** goes through the ordinary + pipeline, with one extra stage under the room lock: the invite is sent + to the invitee's server to countersign (`/invite`, v2), and the copy with + both signatures is the one persisted. + +And the other side of each: + +- `make_join` builds the template from the room's current state and runs + the rules over a trial build first, so that a banned user hears so before + signing anything. Under the restricted join rule it names a local user to + authorise the join, and `send_join` countersigns it. +- `send_join` and `send_leave` take the event through the same intake as + any other, and mark it to be **relayed**: the joining server is not in + the room yet and relies on this one to tell everybody else. +- `/invite` countersigns an invite to a local user. A room this server is + in takes the invite in as an ordinary event. One it is not in gets it as + an out-of-band membership — an outlier that is nonetheless the invitee's + membership in the room's current state, which is all `/sync` needs — and + the stripped state the inviter sent along is kept in + `roomserver_invite_state` for the invite list, until the invite is + answered. + +## The outbound queue + +`federationapi.Queue` sends this server's events and EDUs to everybody who +should have them. + +The roomserver's emit stage calls it with the room lock held, so that call +only appends to an in-memory list. A dispatcher drains the list, works out +each event's destinations — the servers with a user joined to the room, +plus, for a membership event, the server of the user it is about, so that +a server whose last user was just kicked hears of it — and writes one row +per destination to `federation_queue`. An event received from another +server is not sent on, because its origin sends it to everybody itself; +the exception is a relayed join or leave, which goes to everybody but its +origin. + +A worker per destination sends the rows in transactions of at most 50 PDUs +and 100 EDUs, and deletes them once the destination has answered. A +failure backs the destination off — ten seconds, doubling to six hours, +with a fifth of jitter — in `federation_destinations`, which the admin API +reports and can reset. A destination later denied by the allow and deny +lists has its backlog dropped rather than sent. + +Rows are deleted only once delivered, so a restart loses nothing that had +reached the database: the queue starts a worker for every destination with +a backlog. Transaction IDs carry the queue's start time, because a +receiving server deduplicates by transaction ID and an ID reused after a +restart with different contents would be answered from its cache. + +Received transactions are recorded with their response in +`federation_inbound_txns`, so that a retransmission is answered the same +way rather than processed twice. + +## EDUs + +| EDU | Out | In | +|---|---|---| +| `m.typing` | a local user's typing, to the room's servers | into the typing tracker, if the user belongs to the sending server | +| `m.receipt` | a local user's public read receipts — never private ones | into the receipts stream, likewise | +| `m.direct_to_device` | to-device messages for remote users, one EDU per server | delivered to local devices | +| `m.presence`, `m.device_list_update` | — | ignored: presence is off by default, device lists are Phase 7 | + +Nothing in an EDU is signed, so the one check there is — the user must +belong to the server that sent it — is what keeps a server from making +anybody appear to type. EDUs about a room are subject to its server ACL. + +The sync API hands EDUs over through `syncapi.Outbound`, and only for what +a local user did: the same calls are how EDUs from other servers come in, +and passing those on would echo them back. + +## Server ACLs + +`m.room.server_acl` is enforced against the origin of everything about a +room: its PDUs, its typing and receipts, its joins and leaves, and its +reads of the room's history. Globs match case-insensitively, without the +port; deny wins over allow; an empty allow list allows nobody; and +`allow_ip_literals: false` refuses servers named by an IP address. An ACL +never applies to the server enforcing it. + +## What other servers may read + +`/event`, `/state`, `/state_ids`, `/event_auth`, `/backfill` and +`/get_missing_events` are answered for a server with a user joined to the +room, or for any server if the room is world-readable — and only if the +room's ACL allows it. Anybody else gets `403` whether or not the event +exists. + +`/state_ids` and `/state` answer with the state *before* the event asked +about, which is what every implementation's client side expects; the +asker adds the event itself if it is a state event. An outlier has no +snapshot, and asking for the state at one is a `404`. + +## Endpoints + +| | | +|---|---| +| `GET /_matrix/key/v2/server` · `POST /_matrix/key/v2/query` · `GET /_matrix/key/v2/query/{server}` | keys; the notary answers for this server only | +| `GET /_matrix/federation/v1/version` | | +| `PUT /v1/send/{txnId}` | transactions of PDUs and EDUs | +| `GET /v1/make_join/{room}/{user}` · `PUT /v1/send_join`, `/v2/send_join` | joins | +| `GET /v1/make_leave/{room}/{user}` · `PUT /v1/send_leave`, `/v2/send_leave` | leaves | +| `PUT /v2/invite/{room}/{event}` | invites | +| `GET /v1/event/{event}` · `/v1/state/{room}` · `/v1/state_ids/{room}` · `/v1/event_auth/{room}/{event}` | reads | +| `GET /v1/backfill/{room}` · `POST /v1/get_missing_events/{room}` | history | +| `GET /v1/query/directory` · `/v1/query/profile` | queries | +| `GET`, `POST /v1/publicRooms` | the public room directory | + +Client-Server endpoints that now reach other servers: `/join` and +`/rooms/{id}/join` for a room this server is not in (by ID or by remote +alias), `/invite` for a remote user, `/leave` to reject a remote invite, +`GET /directory/room/{alias}` for a remote alias, `/publicRooms?server=`, +and `GET /profile/{user}` for a remote user. + +## Known gaps + +- **Backfill only fills in history below the room.** A gap higher up — a + live event whose prev_events were closed with `/state_ids` rather than + fetched — stays a gap. And the end of a server's history is not + remembered: a room whose servers cannot supply anything before its + oldest event is asked again each time a client scrolls to the top. +- **Backfilled redactions are not applied** to earlier backfilled events; + a server sends a redacted event in its redacted form anyway, but + `unsigned.redacted_because` is not filled in. +- **No partial-state joins** (MSC3706). Joining a room of tens of + thousands of members means receiving, verifying and checking its whole + state before the join completes, which takes minutes. +- **No key rotation.** The key file holds one key; old keys are not + served as `old_verify_keys`, so rotating the key would invalidate this + server's old events for anybody who has not cached the old key. +- **Rejected events are not remembered.** A rejected event is not stored, + so if it is sent again it is fetched and judged again. +- **Soft-failed events are never re-evaluated**, and a redaction whose + target arrives later is not applied when the target arrives. +- **Room versions 1 and 2** — event IDs carried in the event — are not + supported, so the oldest rooms on the network cannot be joined. +- **Presence and device list EDUs** are ignored on the way in and not sent + (presence is off by default; device lists are Phase 7). +- **Third-party invites** across servers (`exchange_third_party_invite`) + and **knocking** over federation (`make_knock`, `send_knock`) are not + implemented. + +## Testing + +| Where | What | +|---|---| +| `internal/federation` | X-Matrix parsing (fuzzed) and signing, discovery against fake DNS and a fake well-known server, the key ring against a fake key server, an authenticated request end to end over TLS, the client refusing while disabled or denied. | +| `internal/roomserver/federated_test.go` | Two and three roomservers in one process, wired to each other's federation methods: joins at versions 10–12, conversation and fork merging, fail-over, refused joins, invites with countersignatures, rejection with and without the inviter reachable, missing events, a gap closed with state, forged and soft-failed events, ACLs, remote aliases and directories, the switch. `backfill_test.go`: a server that joined late scrolling back through the whole room at versions 10–12, the state inside the history, backfill with the switch off and under `joined` visibility. | +| `internal/federationapi` | The queue: fan-out and relay rules, backoff and retry, transactions within the limits, a restart with a backlog, the policy, EDUs. | +| `internal/setup/phase5_test.go` | Two complete servers over HTTPS: a join by remote alias, messages both ways through the queues, a remote profile, federation switched off with a message held and delivered after it is switched back on, the admin view of it; the TLS listener and the module following the switch. | +| Complement | `make complement-federation` — see [server/test/complement](../server/test/complement/README.md). | diff --git a/docs/rooms.md b/docs/rooms.md index ee206ac..bc0c3f3 100644 --- a/docs/rooms.md +++ b/docs/rooms.md @@ -2,9 +2,9 @@ `internal/roomserver` and the room half of `internal/storage`: PLAN.md Phase 3. This is what turns the event core of Phase 2 into rooms a client -can use — locally. Nothing here talks to another server; federation is -Phase 5, and every operation that would need it is refused with a message -that says so rather than a bare `M_FORBIDDEN`. +can use. Phase 3 built it local only; Phase 5 taught the same pipeline to +take in events from other servers and to join, leave and invite across +them, which [federation.md](federation.md) describes. ## Where the decisions live @@ -65,7 +65,7 @@ With that lock, a Phase 3 room cannot fork, so the resolution path is unreachable from outside. It is implemented anyway because the alternative is a pipeline that authorises against one arbitrary branch of a fork without saying so — and because `stateAfter`, which it needs, is also what -federation will need in Phase 5. +federation needs, and since Phase 5 uses. ### Soft failure @@ -107,7 +107,9 @@ handed out by a counter row inside the write transaction. Neither `AUTOINCREMENT` nor a sequence would do: the two engines spell those differently, and a PostgreSQL sequence is explicitly not transactional, so a rolled-back write would leave a hole that `/sync` would have to be taught to -skip. +skip. History backfilled from other servers is numbered by a second counter +that runs downwards from -1, so that it sorts below everything already +held; see [federation.md](federation.md#history-backfill). **State snapshots** are stored per event and deduplicated by a hash of their contents. That is what makes a per-event snapshot affordable: every message @@ -136,10 +138,10 @@ PLAN.md section 3, enforced here: | Operation | While federation is off | |---|---| -| `/join` a room this server does not have | `403`, naming federation | -| Inviting `@user:other.example` | `403`, naming federation | -| Resolving `#alias:other.example` | `403`, naming federation | -| `publicRooms?server=other.example` | `403`, naming federation | +| `/join` a room this server is not in | `403`, naming federation; with it on, a join through a server that is | +| Inviting `@user:other.example` | `403`, naming federation; with it on, the invite is countersigned by their server | +| Resolving `#alias:other.example` | `403`, naming federation; with it on, the owning server is asked | +| `publicRooms?server=other.example` | `403`, naming federation; with it on, the other server is asked | | Room and event IDs | unchanged — a room created offline federates normally if federation is later turned on | | Signing | unchanged — every event is signed with the server key | @@ -147,10 +149,11 @@ The messages are explicit on purpose. `M_FORBIDDEN` alone sends a user to their operator and the operator hunting through power levels. Restricted joins are the interesting case. The rule needs a signature from -the authorising user's server checked, so it normally needs a key fetcher — -but while federation is off every room is local, this server's own key is in -hand, and the rule can be applied in full. A join authorised via a *remote* -user is refused, like everything else that would need another server. +the authorising user's server checked. This server's own signatures are +checked against the key in hand, so a restricted join authorised by a local +user works whether or not federation is on; one authorised by a remote user +needs the key ring, and while federation is off it is refused like +everything else that would need another server. ## Known gaps diff --git a/docs/sync.md b/docs/sync.md index b13e229..ecd37d8 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -71,6 +71,11 @@ The spec says only that the token is opaque, and clients treat it that way. It is worth being readable anyway: a token is the first thing in every bug report about `/sync`. +The PDU position is the one that can be negative: `/messages` pages back +into history backfilled from other servers, which is numbered downwards +from -1 (see [federation.md](federation.md#history-backfill)), and its +tokens name positions there — `s-12`. The other streams start at zero. + `/messages` and a room's `prev_batch` hand out a PDU-only token. Both are about room events alone, and a token from either is a valid `?since=` for `/sync`, where the other streams then read as unseen — the conservative @@ -112,6 +117,12 @@ type TimelineRequest struct { Low exclusive and High inclusive, **in both directions**. Direction decides only which end the limit trims and which order the store reads in. +Zero is "unbounded" at either end, and is never a position: live events +count up from 1 and backfilled history counts down from -1. So "is this +bound set" is `!= 0`, never `> 0`, and the position just below an event is +`storage.PositionBefore` rather than a subtraction — below 1 is -1, because +a token of zero handed back as a High bound would read the whole room again. + Spelling the bounds as "from" and "to" instead — which is how the Client-Server API spells them — would mean each of them being the low bound in one direction and the high bound in the other, which is exactly the @@ -354,8 +365,8 @@ for a key share is both useless and a leak. A message to a user of another server is dropped rather than refused. A client encrypting for a room with remote members sends one message per device in one request, and failing the request because federation is off -would stop it from delivering to the local devices as well. Phase 5 routes -them. +would stop it from delivering to the local devices as well. With federation +on, they go to the federation queue as `m.direct_to_device` EDUs instead. ### Presence @@ -391,7 +402,7 @@ no timestamp and have to decide what that meant. |---|---|---| | `unread_notifications`, `unread_thread_notifications` | push rules | 8 | | `device_lists.changed` / `.left`, `device_one_time_keys_count` | the keyserver | 7 | -| Remote presence, remote receipts, remote typing | federation | 5 | +| Remote presence | presence over federation | — | | Threads, and threaded receipts beyond storing `thread_id` | threads | 10 | | `contains_url`, `event_fields` | nothing; they are parsed and ignored | — | | Per-event history visibility | state storage (open question 1) | — | diff --git a/server/internal/adminapi/adminapi.go b/server/internal/adminapi/adminapi.go index 8d23bec..7c3e27b 100644 --- a/server/internal/adminapi/adminapi.go +++ b/server/internal/adminapi/adminapi.go @@ -39,6 +39,7 @@ type AdminAPI struct { settings *settings.Service federation *federation.Gate database storage.Database + fedStatus FederationStatus log *slog.Logger } @@ -55,6 +56,10 @@ type Options struct { Settings *settings.Service Federation *federation.Gate + // FederationStatus reports on the federation module. Nil on a server + // without one, and the destinations endpoints then say so. + FederationStatus FederationStatus + // Database is used for the driver name the dashboard shows and for the // migration state; the admin API does not write to it directly. Database storage.Database @@ -85,6 +90,7 @@ func New(opts Options) (*AdminAPI, error) { settings: opts.Settings, federation: opts.Federation, database: opts.Database, + fedStatus: opts.FederationStatus, log: opts.Logger, }, nil } @@ -93,6 +99,8 @@ func New(opts Options) (*AdminAPI, error) { func (a *AdminAPI) RegisterRoutes(r *httputil.Router) { a.handle(r, "GET "+Prefix+"/federation", a.handleGetFederation) a.handle(r, "PUT "+Prefix+"/federation", a.handlePutFederation) + a.handle(r, "GET "+Prefix+"/federation/destinations", a.handleGetDestinations) + a.handle(r, "POST "+Prefix+"/federation/destinations/{serverName}/reset", a.handleResetDestination) a.handle(r, "GET "+Prefix+"/registration", a.handleGetRegistration) a.handle(r, "PUT "+Prefix+"/registration", a.handlePutRegistration) a.handle(r, "GET "+Prefix+"/users", a.handleGetUsers) diff --git a/server/internal/adminapi/federation.go b/server/internal/adminapi/federation.go new file mode 100644 index 0000000..ef8b877 --- /dev/null +++ b/server/internal/adminapi/federation.go @@ -0,0 +1,102 @@ +package adminapi + +import ( + "context" + "net/http" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/httputil" + "github.com/dariuszjaniak/warp/server/internal/storage" + "github.com/dariuszjaniak/warp/server/internal/userapi" +) + +// FederationStatus is what the admin API can see of the running federation +// module: whether it is up, and the delivery health of every server it +// sends to. +type FederationStatus interface { + Running() bool + Destinations(ctx context.Context) ([]storage.DestinationSummary, error) + ResetBackoff(ctx context.Context, serverName string) error +} + +// destinationResponse is one server in the destinations listing. +type destinationResponse struct { + ServerName string `json:"server_name"` + + // Failures is how many sends in a row have failed; zero is healthy. + Failures int `json:"failures"` + + // RetryAtMS is when the next attempt is due, or 0 for now. + RetryAtMS int64 `json:"retry_at_ms"` + LastSuccessMS int64 `json:"last_success_ms"` + LastFailureMS int64 `json:"last_failure_ms"` + LastError string `json:"last_error,omitempty"` + + // QueuedPDUs and QueuedEDUs are the queue depth: what this server + // still owes the destination. + QueuedPDUs int `json:"queued_pdus"` + QueuedEDUs int `json:"queued_edus"` +} + +type destinationsResponse struct { + Running bool `json:"running"` + Destinations []destinationResponse `json:"destinations"` +} + +// requireFederationStatus refuses on a server built without federation. +func (a *AdminAPI) requireFederationStatus() error { + if a.fedStatus == nil { + return httputil.NotFound("This server was started without federation (it has no signing key)") + } + return nil +} + +// handleGetDestinations implements GET /_warp/admin/v1/federation/destinations: +// per-server queue depth and backoff, the panel's federation health view. +func (a *AdminAPI) handleGetDestinations(w http.ResponseWriter, r *http.Request, _ userapi.Authenticated) error { + if err := a.requireFederationStatus(); err != nil { + return err + } + all, err := a.fedStatus.Destinations(r.Context()) + if err != nil { + return err + } + resp := destinationsResponse{ + Running: a.fedStatus.Running(), + Destinations: make([]destinationResponse, 0, len(all)), + } + for _, d := range all { + resp.Destinations = append(resp.Destinations, destinationResponse{ + ServerName: d.ServerName, + Failures: d.Failures, + RetryAtMS: d.RetryAtMS, + LastSuccessMS: d.LastSuccessMS, + LastFailureMS: d.LastFailureMS, + LastError: d.LastError, + QueuedPDUs: d.QueuedPDUs, + QueuedEDUs: d.QueuedEDUs, + }) + } + httputil.NoCache(w) + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} + +// handleResetDestination implements +// POST /_warp/admin/v1/federation/destinations/{serverName}/reset: forget a +// server's backoff and try it now, for an operator who knows it is back. +func (a *AdminAPI) handleResetDestination(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + if err := a.requireFederationStatus(); err != nil { + return err + } + serverName := r.PathValue("serverName") + if _, err := federation.ParseServerName(serverName); err != nil { + return httputil.InvalidParam("%q is not a server name", serverName) + } + if err := a.fedStatus.ResetBackoff(r.Context(), serverName); err != nil { + return err + } + a.log.Info("federation backoff reset through the admin API", "server_name", serverName, "by", auth.UserID) + httputil.WriteJSON(w, http.StatusOK, struct{}{}) + return nil +} diff --git a/server/internal/clientapi/clientapi.go b/server/internal/clientapi/clientapi.go index b30a3ae..2a2e5e3 100644 --- a/server/internal/clientapi/clientapi.go +++ b/server/internal/clientapi/clientapi.go @@ -36,6 +36,7 @@ type ClientAPI struct { sync *syncapi.API settings *settings.Service federation *federation.Gate + remoteProfile RemoteProfiles registration config.Registration log *slog.Logger now func() time.Time @@ -91,6 +92,10 @@ type Options struct { // Federation gates the endpoints that would need a remote server. Federation *federation.Gate + // RemoteProfiles asks another server for one of its users' profile. + // Nil means remote profiles cannot be looked up at all. + RemoteProfiles RemoteProfiles + // Registration is the file configuration of the registration flow. Registration config.Registration @@ -135,6 +140,7 @@ func New(opts Options) (*ClientAPI, error) { sync: opts.Sync, settings: opts.Settings, federation: opts.Federation, + remoteProfile: opts.RemoteProfiles, registration: opts.Registration, log: opts.Logger, now: opts.Now, diff --git a/server/internal/clientapi/profile.go b/server/internal/clientapi/profile.go index 0211eba..5c7ddb1 100644 --- a/server/internal/clientapi/profile.go +++ b/server/internal/clientapi/profile.go @@ -1,11 +1,13 @@ package clientapi import ( + "context" "errors" "net/http" "github.com/dariuszjaniak/warp/server/internal/federation" "github.com/dariuszjaniak/warp/server/internal/httputil" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" "github.com/dariuszjaniak/warp/server/internal/storage" "github.com/dariuszjaniak/warp/server/internal/userapi" ) @@ -59,24 +61,29 @@ func (c *ClientAPI) handleGetAvatarURL(w http.ResponseWriter, r *http.Request) e return nil } -// lookupProfile resolves the user ID in the path to a local account. +// RemoteProfiles asks another server for a profile. *federation.Client +// satisfies it. +type RemoteProfiles interface { + QueryProfile(ctx context.Context, destination, userID, field string) (map[string]any, error) +} + +// lookupProfile resolves the user ID in the path to an account: a local +// one from the database, a remote one by asking its server. // // A user ID on another server is where the federation switch shows up in the // Client-Server API: with federation off there is no way to answer and the // spec's M_FORBIDDEN is the honest reply, as PLAN.md section 3 requires. -// With federation on, the lookup itself is Phase 5 work. func (c *ClientAPI) lookupProfile(r *http.Request) (storage.Account, error) { userID := r.PathValue("userID") localpart, err := c.users.ParseLocalUserID(userID) switch { case errors.Is(err, userapi.ErrForeignServer): - if !c.federation.Enabled() { + if !c.federation.Enabled() || c.remoteProfile == nil { return storage.Account{}, federation.RejectClientRequest( "looking up the profile of a user on another server") } - return storage.Account{}, httputil.NotFound( - "Profiles of users on other servers cannot be fetched yet") + return c.lookupRemoteProfile(r.Context(), userID) case err != nil: return storage.Account{}, httputil.InvalidParam("%s is not a valid user ID", userID) } @@ -155,3 +162,34 @@ func profileWriteError(err error) error { } return err } + +// lookupRemoteProfile asks a user's own server for their profile. +// +// Nothing is cached. A profile changes whenever its owner likes, and a +// cached copy would need invalidating by an EDU this server does not +// receive; the membership events of the rooms a remote user shares with +// ours carry their display name already, which is where clients look for +// it most of the time. +func (c *ClientAPI) lookupRemoteProfile(ctx context.Context, userID string) (storage.Account, error) { + server, err := id.ServerNameOf(userID) + if err != nil { + return storage.Account{}, httputil.InvalidParam("%s is not a valid user ID", userID) + } + profile, err := c.remoteProfile.QueryProfile(ctx, server, userID, "") + switch { + case federation.IsNotFound(err): + return storage.Account{}, httputil.NotFound("Profile not found") + case err != nil: + c.log.Info("remote profile lookup failed", "user_id", userID, "error", err) + return storage.Account{}, httputil.NewError(http.StatusBadGateway, httputil.ErrCodeUnknown, + "%s could not be asked for the profile", server) + } + account := storage.Account{} + if name, ok := profile["displayname"].(string); ok { + account.DisplayName = name + } + if avatar, ok := profile["avatar_url"].(string); ok { + account.AvatarURL = avatar + } + return account, nil +} diff --git a/server/internal/clientapi/profile_test.go b/server/internal/clientapi/profile_test.go index be4f7fc..1fe93aa 100644 --- a/server/internal/clientapi/profile_test.go +++ b/server/internal/clientapi/profile_test.go @@ -1,9 +1,13 @@ package clientapi_test import ( + "context" "net/http" "strings" "testing" + + "github.com/dariuszjaniak/warp/server/internal/clientapi" + "github.com/dariuszjaniak/warp/server/internal/federation" ) const profilePath = "/_matrix/client/v3/profile" @@ -120,7 +124,8 @@ func TestProfileRejectsMalformedUserID(t *testing.T) { // no way to answer, and the spec's M_FORBIDDEN with a clear message is the // honest reply. func TestProfileOfRemoteUserHonoursTheFederationSwitch(t *testing.T) { - ts := newTestServer(t, nil) + remote := fakeRemoteProfiles{"@bob:other.example": {"displayname": "Bob Remote"}} + ts := newTestServer(t, func(o *clientapi.Options) { o.RemoteProfiles = remote }) rec := ts.get(profilePath+"/@bob:other.example", "") expectError(t, rec, http.StatusForbidden, "M_FORBIDDEN") @@ -128,15 +133,33 @@ func TestProfileOfRemoteUserHonoursTheFederationSwitch(t *testing.T) { t.Errorf("the message should say why, got %q", msg) } - // With the switch on it is no longer a refusal, only something that is - // not implemented until Phase 5. + // With the switch on the user's own server is asked. if err := ts.settings.SetFederationEnabled(t.Context(), true); err != nil { t.Fatalf("SetFederationEnabled: %v", err) } - expectError(t, ts.get(profilePath+"/@bob:other.example", ""), + rec = ts.get(profilePath+"/@bob:other.example", "") + if rec.Code != http.StatusOK { + t.Fatalf("remote profile status = %d (body %s)", rec.Code, rec.Body.String()) + } + if name, _ := jsonBody(t, rec)["displayname"].(string); name != "Bob Remote" { + t.Errorf("remote display name = %q", name) + } + expectError(t, ts.get(profilePath+"/@nobody:other.example", ""), http.StatusNotFound, "M_NOT_FOUND") } +// fakeRemoteProfiles answers profile queries from a table, as the remote +// server would. +type fakeRemoteProfiles map[string]map[string]any + +func (f fakeRemoteProfiles) QueryProfile(_ context.Context, _, userID, _ string) (map[string]any, error) { + profile, ok := f[userID] + if !ok { + return nil, &federation.HTTPError{Status: http.StatusNotFound, Code: "M_NOT_FOUND"} + } + return profile, nil +} + func TestProfileRejectsOverlongDisplayName(t *testing.T) { ts := newTestServer(t, nil) acc := ts.register("alice", "superuser") diff --git a/server/internal/clientapi/rooms.go b/server/internal/clientapi/rooms.go index 4d9b254..a9f897e 100644 --- a/server/internal/clientapi/rooms.go +++ b/server/internal/clientapi/rooms.go @@ -46,6 +46,14 @@ func roomError(err error) error { case errors.Is(err, roomserver.ErrFederationDisabled): return httputil.Forbidden("%s", err.Error()) + case errors.Is(err, roomserver.ErrIncompatibleRoomVersion): + return httputil.NewError(http.StatusBadRequest, "M_INCOMPATIBLE_ROOM_VERSION", "%s", err.Error()) + case errors.Is(err, roomserver.ErrNotInRoom): + return httputil.Forbidden("%s", err.Error()) + case errors.Is(err, roomserver.ErrRemote), errors.Is(err, roomserver.ErrBadEvent): + // Another server failed us; the client can do nothing about it but + // try again, or try another server. + return httputil.NewError(http.StatusBadGateway, httputil.ErrCodeUnknown, "%s", err.Error()) case errors.Is(err, roomserver.ErrRejected), errors.Is(err, roomserver.ErrForbidden): return httputil.Forbidden("%s", err.Error()) case errors.Is(err, roomserver.ErrInvalidContent): diff --git a/server/internal/clientapi/sync_test.go b/server/internal/clientapi/sync_test.go index 664e308..60a4303 100644 --- a/server/internal/clientapi/sync_test.go +++ b/server/internal/clientapi/sync_test.go @@ -257,7 +257,10 @@ func TestSyncRejectsAnUnreadableToken(t *testing.T) { ts := newTestServer(t, nil) alice := ts.register("alice", "correct-horse-battery-staple") - for _, since := range []string{"nonsense", "s-1", "sx", "s1_2_3_4_5_6"} { + // "s-1" is not here: a negative event position is a /messages token + // from inside backfilled history, and a valid ?since=. Only the event + // stream goes below zero. + for _, since := range []string{"nonsense", "s1_-1", "sx", "s1_2_3_4_5_6"} { rec := ts.get("/_matrix/client/v3/sync?since="+url.QueryEscape(since), alice.AccessToken) expectError(t, rec, http.StatusBadRequest, "M_INVALID_PARAM") } diff --git a/server/internal/config/config.go b/server/internal/config/config.go index 795c143..3c1e2a5 100644 --- a/server/internal/config/config.go +++ b/server/internal/config/config.go @@ -72,6 +72,20 @@ type Listen struct { // ShutdownTimeout bounds graceful shutdown before connections are cut. ShutdownTimeout time.Duration `yaml:"shutdown_timeout" env:"WARP_SHUTDOWN_TIMEOUT"` + + // FederationAddress, when set, opens a second listener serving the + // same API over TLS. It is the port other servers connect to when + // nothing delegates them elsewhere — 8448 — and the one deployment + // shape where Warp terminates TLS itself rather than sitting behind a + // reverse proxy. Empty means no TLS listener: federation traffic then + // arrives through whatever the well-known document or SRV record + // points at. + FederationAddress string `yaml:"federation_address" env:"WARP_LISTEN_FEDERATION_ADDRESS"` + + // TLSCertFile and TLSKeyFile are the PEM certificate chain and key for + // the federation listener. Both are required when it is enabled. + TLSCertFile string `yaml:"tls_cert_file" env:"WARP_TLS_CERT_FILE"` + TLSKeyFile string `yaml:"tls_key_file" env:"WARP_TLS_KEY_FILE"` } // Database selects and configures the storage driver. @@ -195,6 +209,24 @@ type Federation struct { // AllowRuntimeToggle lets the admin API flip Enabled without a restart. // When true the persisted setting wins over the value above on boot. AllowRuntimeToggle bool `yaml:"allow_runtime_toggle" env:"WARP_FEDERATION_ALLOW_RUNTIME_TOGGLE"` + + // TrustedKeyServers are the notaries asked for another server's + // signing key when that server cannot be reached directly. Empty — + // the default — means keys are only ever fetched from their own + // server, which trusts nobody but requires everybody to be online. + TrustedKeyServers []string `yaml:"trusted_key_servers" env:"WARP_FEDERATION_TRUSTED_KEY_SERVERS"` + + // KeyValidityPeriod is how long this server's key response tells + // others they may cache its key. Shorter means a rotated key is + // noticed sooner; longer means fewer fetches. The spec caps what a + // receiver believes at a week. + KeyValidityPeriod time.Duration `yaml:"key_validity_period" env:"WARP_FEDERATION_KEY_VALIDITY_PERIOD"` + + // TLSCAFile adds a PEM bundle of certificate authorities to the system + // roots for outbound federation, for a private federation whose + // servers have certificates from an internal CA. The system roots + // still apply. + TLSCAFile string `yaml:"tls_ca_file" env:"WARP_FEDERATION_TLS_CA_FILE"` } // Registration controls open registration. @@ -295,6 +327,7 @@ func Default() *Config { Federation: Federation{ Enabled: false, AllowRuntimeToggle: true, + KeyValidityPeriod: 24 * time.Hour, }, Registration: Registration{ Enabled: false, @@ -429,6 +462,17 @@ func (c *Config) Validate() error { bad("federation.denylist[%d]: %w", i, err) } } + for i, s := range c.Federation.TrustedKeyServers { + if err := validateServerName(s); err != nil { + bad("federation.trusted_key_servers[%d]: %w", i, err) + } + } + if c.Federation.KeyValidityPeriod <= 0 || c.Federation.KeyValidityPeriod > 7*24*time.Hour { + bad("federation.key_validity_period: must be positive and at most a week (168h)") + } + if c.Listen.FederationAddress != "" && (c.Listen.TLSCertFile == "" || c.Listen.TLSKeyFile == "") { + bad("listen.federation_address: needs listen.tls_cert_file and listen.tls_key_file") + } return errors.Join(errs...) } diff --git a/server/internal/federation/client.go b/server/internal/federation/client.go new file mode 100644 index 0000000..7a0331d --- /dev/null +++ b/server/internal/federation/client.go @@ -0,0 +1,528 @@ +package federation + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/dariuszjaniak/warp/server/internal/buildinfo" + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" +) + +// Errors an outbound request can fail with before anything is sent. +var ( + // ErrDisabled means federation is switched off. Every outbound request + // checks the switch itself, so that a component that missed the + // toggle still cannot reach another server. + ErrDisabled = errors.New("federation is disabled") + + // ErrNotAllowed means the destination is excluded by the allow and + // deny lists. + ErrNotAllowed = errors.New("federation with that server is not allowed") + + // ErrSelf means a request was addressed to this server. Asking + // ourselves over the network can only be a bug in the caller, and on + // a server whose name does not resolve to itself it would ask + // somebody else. + ErrSelf = errors.New("refusing to send a federation request to this server") +) + +// HTTPError is a non-2xx answer from another server. +type HTTPError struct { + Status int + Code string + Message string +} + +func (e *HTTPError) Error() string { + if e.Code != "" { + return fmt.Sprintf("remote server answered %d %s: %s", e.Status, e.Code, e.Message) + } + return fmt.Sprintf("remote server answered %d", e.Status) +} + +// IsNotFound reports a 404 from another server. +func IsNotFound(err error) bool { + var httpErr *HTTPError + return errors.As(err, &httpErr) && httpErr.Status == http.StatusNotFound +} + +// Response size limits. A send_join answer for a large room is the room's +// entire state and auth chain, which for a public room of tens of +// thousands of members is hundreds of megabytes; nothing else comes close. +const ( + defaultMaxResponse = 16 << 20 + largeMaxResponse = 1 << 30 +) + +// Request timeouts. Joining a large room makes the resident server +// assemble and sign that whole answer first, which takes minutes. +const ( + defaultRequestTimeout = time.Minute + longRequestTimeout = 10 * time.Minute +) + +// ResolveFunc turns a server name into where to connect. +type ResolveFunc func(ctx context.Context, serverName string) (Resolved, error) + +// ClientOptions configure a Client. +type ClientOptions struct { + // ServerName is this server's name: the origin of every request. + ServerName string + + // Key signs every request. + Key *signing.KeyPair + + // Resolve is server discovery. Nil uses a Resolver. + Resolve ResolveFunc + + // TLSConfig is the client TLS configuration, for tests and for + // deployments with a private CA. Nil uses the system roots. + TLSConfig *tls.Config + + // Enabled reports the live federation switch. Nil means off. + Enabled func() bool + + // Policy filters destinations. Nil allows any. + Policy *Policy +} + +// Client sends signed Server-Server API requests. +type Client struct { + serverName string + key *signing.KeyPair + resolve ResolveFunc + http *http.Client + enabled func() bool + policy *Policy + userAgent string +} + +// dialTargetKey carries the resolved addresses of a request down to the +// dialer, which is the only place they are needed: the URL holds the Host +// the certificate is checked against, and the address the connection goes +// to can be something else entirely. +type dialTargetKey struct{} + +// NewClient builds a client. +func NewClient(opts ClientOptions) *Client { + dialer := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second} + tlsConfig := opts.TLSConfig + if tlsConfig == nil { + tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12} + } + transport := &http.Transport{ + Proxy: nil, + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + targets, _ := ctx.Value(dialTargetKey{}).([]string) + if len(targets) == 0 { + return dialer.DialContext(ctx, network, addr) + } + var lastErr error + for _, target := range targets { + conn, err := dialer.DialContext(ctx, network, target) + if err == nil { + return conn, nil + } + lastErr = err + } + return nil, lastErr + }, + TLSClientConfig: tlsConfig, + TLSHandshakeTimeout: 15 * time.Second, + ResponseHeaderTimeout: 0, // bounded by the request context instead + MaxIdleConns: 200, + MaxIdleConnsPerHost: 4, + IdleConnTimeout: 90 * time.Second, + ForceAttemptHTTP2: true, + } + + if opts.Enabled == nil { + opts.Enabled = func() bool { return false } + } + if opts.Policy == nil { + opts.Policy = NewPolicy(nil, nil) + } + c := &Client{ + serverName: opts.ServerName, + key: opts.Key, + resolve: opts.Resolve, + http: &http.Client{ + Transport: transport, + // Federation endpoints do not redirect; following one would + // send a signed request somewhere it was not addressed. + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }, + enabled: opts.Enabled, + policy: opts.Policy, + userAgent: buildinfo.UserAgent(), + } + if c.resolve == nil { + // The well-known document is fetched with ordinary HTTPS and the + // same roots as everything else. + resolver := NewResolver(&http.Client{ + Timeout: 30 * time.Second, + Transport: &http.Transport{TLSClientConfig: tlsConfig, Proxy: nil}, + }) + c.resolve = resolver.Resolve + } + return c +} + +// ServerName returns the origin this client signs as. +func (c *Client) ServerName() string { return c.serverName } + +// check refuses a destination before anything goes on the wire. +func (c *Client) check(destination string) error { + switch { + case !c.enabled(): + return ErrDisabled + case destination == c.serverName: + return ErrSelf + case !c.policy.Allowed(destination): + return fmt.Errorf("%w: %s", ErrNotAllowed, destination) + } + return nil +} + +// request describes one call. +type request struct { + method string + destination string + path string // already escaped + query url.Values + body any + signed bool + maxBytes int64 + timeout time.Duration +} + +// do performs a request and decodes a JSON answer into out (which may be +// nil). +func (c *Client) do(ctx context.Context, req request, out any) error { + if err := c.check(req.destination); err != nil { + return err + } + resolved, err := c.resolve(ctx, req.destination) + if err != nil { + return fmt.Errorf("resolve %s: %w", req.destination, err) + } + + timeout := req.timeout + if timeout == 0 { + timeout = defaultRequestTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + ctx = context.WithValue(ctx, dialTargetKey{}, resolved.Addresses) + + var content []byte + if req.body != nil { + if content, err = json.Marshal(req.body); err != nil { + return fmt.Errorf("encode request: %w", err) + } + } + + target := "https://" + resolved.Host + req.path + if len(req.query) > 0 { + target += "?" + req.query.Encode() + } + httpReq, err := http.NewRequestWithContext(ctx, req.method, target, bytes.NewReader(content)) + if err != nil { + return fmt.Errorf("build request: %w", err) + } + if content == nil { + httpReq.Body = http.NoBody + } else { + httpReq.Header.Set("Content-Type", "application/json") + } + httpReq.Header.Set("User-Agent", c.userAgent) + httpReq.Header.Set("Accept", "application/json") + + if req.signed { + // The signature covers the request target exactly as it goes on + // the wire, so it is taken from the built request rather than + // from the path this function was handed. + x, err := SignRequest(c.key, c.serverName, req.destination, req.method, + httpReq.URL.RequestURI(), content) + if err != nil { + return fmt.Errorf("sign request: %w", err) + } + httpReq.Header.Set("Authorization", x.String()) + } + + resp, err := c.http.Do(httpReq) + if err != nil { + return fmt.Errorf("%s %s%s: %w", req.method, req.destination, req.path, err) + } + defer func() { _ = resp.Body.Close() }() + + limit := req.maxBytes + if limit == 0 { + limit = defaultMaxResponse + } + body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1)) + if err != nil { + return fmt.Errorf("%s %s%s: read response: %w", req.method, req.destination, req.path, err) + } + if int64(len(body)) > limit { + return fmt.Errorf("%s %s%s: response is larger than %d bytes", req.method, req.destination, req.path, limit) + } + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + httpErr := &HTTPError{Status: resp.StatusCode} + var envelope struct { + Code string `json:"errcode"` + Message string `json:"error"` + } + if json.Unmarshal(body, &envelope) == nil { + httpErr.Code, httpErr.Message = envelope.Code, envelope.Message + } + return httpErr + } + if out == nil { + return nil + } + if err := json.Unmarshal(body, out); err != nil { + return fmt.Errorf("%s %s%s: decode response: %w", req.method, req.destination, req.path, err) + } + return nil +} + +// esc escapes one path segment. Room, event and user IDs carry sigils and +// colons, which are legal in a path; slashes would not be, and an event ID +// of room versions 3 and before can contain one. +func esc(s string) string { return url.PathEscape(s) } + +// ServerKeys fetches a server's own key response, unsigned and unverified: +// verifying it is the key ring's job. +func (c *Client) ServerKeys(ctx context.Context, serverName string) (json.RawMessage, error) { + var out json.RawMessage + err := c.do(ctx, request{method: http.MethodGet, destination: serverName, path: "/_matrix/key/v2/server"}, &out) + return out, err +} + +// QueryKeys asks a notary for other servers' keys. +func (c *Client) QueryKeys(ctx context.Context, notary string, query map[string]map[string]KeyCriteria) ([]json.RawMessage, error) { + var out struct { + ServerKeys []json.RawMessage `json:"server_keys"` + } + err := c.do(ctx, request{ + method: http.MethodPost, destination: notary, path: "/_matrix/key/v2/query", + body: map[string]any{"server_keys": query}, + }, &out) + return out.ServerKeys, err +} + +// KeyCriteria is one key asked of a notary. +type KeyCriteria struct { + MinimumValidUntilTS int64 `json:"minimum_valid_until_ts,omitempty"` +} + +// SendTransaction delivers a transaction. +func (c *Client) SendTransaction(ctx context.Context, destination, txnID string, txn Transaction) (SendTransactionResponse, error) { + var out SendTransactionResponse + err := c.do(ctx, request{ + method: http.MethodPut, destination: destination, signed: true, + path: "/_matrix/federation/v1/send/" + esc(txnID), body: txn, + }, &out) + return out, err +} + +// MakeJoin asks a resident server for a join event template. +func (c *Client) MakeJoin(ctx context.Context, destination, roomID, userID string, versions []string) (MakeJoinResponse, error) { + query := url.Values{} + for _, v := range versions { + query.Add("ver", v) + } + var out MakeJoinResponse + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/make_join/" + esc(roomID) + "/" + esc(userID), + query: query, + }, &out) + return out, err +} + +// SendJoin submits a signed join and receives the room's state. +func (c *Client) SendJoin(ctx context.Context, destination, roomID, eventID string, joinEvent json.RawMessage) (SendJoinResponse, error) { + var out SendJoinResponse + err := c.do(ctx, request{ + method: http.MethodPut, destination: destination, signed: true, + path: "/_matrix/federation/v2/send_join/" + esc(roomID) + "/" + esc(eventID), + body: joinEvent, + maxBytes: largeMaxResponse, + timeout: longRequestTimeout, + }, &out) + return out, err +} + +// MakeLeave asks a resident server for a leave event template. +func (c *Client) MakeLeave(ctx context.Context, destination, roomID, userID string) (MakeJoinResponse, error) { + var out MakeJoinResponse + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/make_leave/" + esc(roomID) + "/" + esc(userID), + }, &out) + return out, err +} + +// SendLeave submits a signed leave. +func (c *Client) SendLeave(ctx context.Context, destination, roomID, eventID string, leaveEvent json.RawMessage) error { + return c.do(ctx, request{ + method: http.MethodPut, destination: destination, signed: true, + path: "/_matrix/federation/v2/send_leave/" + esc(roomID) + "/" + esc(eventID), + body: leaveEvent, + }, nil) +} + +// Invite asks the invitee's server to countersign an invite. +func (c *Client) Invite(ctx context.Context, destination, roomID, eventID string, req InviteRequest) (InviteResponse, error) { + var out InviteResponse + err := c.do(ctx, request{ + method: http.MethodPut, destination: destination, signed: true, + path: "/_matrix/federation/v2/invite/" + esc(roomID) + "/" + esc(eventID), + body: req, + }, &out) + return out, err +} + +// Event fetches one event. +func (c *Client) Event(ctx context.Context, destination, eventID string) (Transaction, error) { + var out Transaction + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/event/" + esc(eventID), + }, &out) + return out, err +} + +// EventAuth fetches the auth chain of an event. +func (c *Client) EventAuth(ctx context.Context, destination, roomID, eventID string) (EventAuthResponse, error) { + var out EventAuthResponse + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/event_auth/" + esc(roomID) + "/" + esc(eventID), + maxBytes: largeMaxResponse, + timeout: longRequestTimeout, + }, &out) + return out, err +} + +// StateIDs fetches the IDs of the room state at an event. +func (c *Client) StateIDs(ctx context.Context, destination, roomID, eventID string) (StateIDsResponse, error) { + var out StateIDsResponse + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/state_ids/" + esc(roomID), + query: url.Values{"event_id": {eventID}}, + maxBytes: largeMaxResponse, + timeout: longRequestTimeout, + }, &out) + return out, err +} + +// State fetches the room state at an event, events and all. +func (c *Client) State(ctx context.Context, destination, roomID, eventID string) (StateResponse, error) { + var out StateResponse + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/state/" + esc(roomID), + query: url.Values{"event_id": {eventID}}, + maxBytes: largeMaxResponse, + timeout: longRequestTimeout, + }, &out) + return out, err +} + +// MissingEvents asks for the events between what this server has and an +// event it was sent. +func (c *Client) MissingEvents(ctx context.Context, destination, roomID string, req MissingEventsRequest) (MissingEventsResponse, error) { + var out MissingEventsResponse + err := c.do(ctx, request{ + method: http.MethodPost, destination: destination, signed: true, + path: "/_matrix/federation/v1/get_missing_events/" + esc(roomID), + body: req, + }, &out) + return out, err +} + +// Backfill asks for history before the given events. +func (c *Client) Backfill(ctx context.Context, destination, roomID string, from []string, limit int) (Transaction, error) { + query := url.Values{"limit": {strconv.Itoa(limit)}} + for _, id := range from { + query.Add("v", id) + } + var out Transaction + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/backfill/" + esc(roomID), + query: query, + }, &out) + return out, err +} + +// QueryProfile asks a server for one of its users' profile. An empty field +// asks for the whole profile. +func (c *Client) QueryProfile(ctx context.Context, destination, userID, field string) (map[string]any, error) { + query := url.Values{"user_id": {userID}} + if field != "" { + query.Set("field", field) + } + var out map[string]any + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/query/profile", query: query, + }, &out) + return out, err +} + +// QueryDirectory resolves an alias on the server that owns it. +func (c *Client) QueryDirectory(ctx context.Context, destination, alias string) (DirectoryResponse, error) { + var out DirectoryResponse + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/query/directory", query: url.Values{"room_alias": {alias}}, + }, &out) + return out, err +} + +// PublicRooms reads another server's public room directory. +func (c *Client) PublicRooms(ctx context.Context, destination string, limit int, since, search string) (json.RawMessage, error) { + body := map[string]any{} + if limit > 0 { + body["limit"] = limit + } + if since != "" { + body["since"] = since + } + if search = strings.TrimSpace(search); search != "" { + body["filter"] = map[string]string{"generic_search_term": search} + } + var out json.RawMessage + err := c.do(ctx, request{ + method: http.MethodPost, destination: destination, signed: true, + path: "/_matrix/federation/v1/publicRooms", body: body, + }, &out) + return out, err +} + +// Version asks a server what software it runs. +func (c *Client) Version(ctx context.Context, destination string) (VersionResponse, error) { + var out VersionResponse + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, + path: "/_matrix/federation/v1/version", + }, &out) + return out, err +} diff --git a/server/internal/federation/discovery.go b/server/internal/federation/discovery.go new file mode 100644 index 0000000..c21360e --- /dev/null +++ b/server/internal/federation/discovery.go @@ -0,0 +1,322 @@ +package federation + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "strconv" + "strings" + "sync" + "time" +) + +// Server discovery: turning a server name into somewhere to connect. +// +// A Matrix server name is not an address. "example.org" may be served from +// matrix.example.org:443 because example.org says so in a well-known +// document, or from whatever an SRV record names, or from example.org:8448 +// when neither exists. The spec's algorithm decides, in a fixed order, and +// the result has two halves that must not be confused: the address to dial, +// and the name the TLS certificate must be valid for (which is also the +// Host header). An SRV target is only ever the first: delegating by SRV +// does not let the target's certificate stand in for the server's. + +// DefaultFederationPort is the port a server name without one is served on +// when nothing delegates it elsewhere. +const DefaultFederationPort = 8448 + +// ErrInvalidServerName is a server name that does not parse. +var ErrInvalidServerName = errors.New("invalid server name") + +// ServerName is a parsed server name: a host, which is a DNS name or an IP +// literal, and an optional port. +type ServerName struct { + Host string + Port int + + // IsIP reports that Host is an IP literal, without the brackets an + // IPv6 literal is written with. + IsIP bool +} + +// ParseServerName parses "host", "host:port", "1.2.3.4:port" or +// "[::1]:port". +func ParseServerName(s string) (ServerName, error) { + if s == "" || len(s) > 255 { + return ServerName{}, fmt.Errorf("%w: %q", ErrInvalidServerName, s) + } + + host, portStr := s, "" + if strings.HasPrefix(s, "[") { + end := strings.IndexByte(s, ']') + if end < 0 { + return ServerName{}, fmt.Errorf("%w: %q has an unterminated IPv6 literal", ErrInvalidServerName, s) + } + host = s[1:end] + rest := s[end+1:] + if rest != "" { + if !strings.HasPrefix(rest, ":") { + return ServerName{}, fmt.Errorf("%w: %q", ErrInvalidServerName, s) + } + portStr = rest[1:] + } + ip := net.ParseIP(host) + if ip == nil || ip.To4() != nil { + return ServerName{}, fmt.Errorf("%w: %q is not an IPv6 literal", ErrInvalidServerName, s) + } + } else if i := strings.LastIndexByte(s, ':'); i >= 0 { + host, portStr = s[:i], s[i+1:] + } + + name := ServerName{Host: host} + if portStr != "" || strings.HasSuffix(s, ":") { + port, err := strconv.Atoi(portStr) + if err != nil || port < 1 || port > 65535 { + return ServerName{}, fmt.Errorf("%w: %q has a bad port", ErrInvalidServerName, s) + } + name.Port = port + } + + if ip := net.ParseIP(host); ip != nil { + if ip.To4() == nil && !strings.HasPrefix(s, "[") { + // A bare IPv6 address with a port is ambiguous, which is why + // the grammar requires the brackets. + return ServerName{}, fmt.Errorf("%w: %q: an IPv6 literal must be bracketed", ErrInvalidServerName, s) + } + name.IsIP = true + return name, nil + } + if !validDNSName(host) { + return ServerName{}, fmt.Errorf("%w: %q is not a DNS name", ErrInvalidServerName, s) + } + return name, nil +} + +// validDNSName applies the spec's dns-name grammar: letters, digits, '-' +// and '.', at most 255 characters. +func validDNSName(s string) bool { + if s == "" || len(s) > 255 { + return false + } + for _, r := range s { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '.': + default: + return false + } + } + return true +} + +// hostPort renders a host and port for dialling, bracketing IPv6. +func hostPort(host string, port int) string { + return net.JoinHostPort(host, strconv.Itoa(port)) +} + +// Resolved is where to send a server's federation traffic. +type Resolved struct { + // Addresses are host:port pairs to try in order. + Addresses []string + + // Host is the Host header and, without its port, the name the TLS + // certificate must be valid for. + Host string +} + +// TLSServerName is the name the server's certificate must match. +func (r Resolved) TLSServerName() string { + if host, _, err := net.SplitHostPort(r.Host); err == nil { + return host + } + return strings.Trim(r.Host, "[]") +} + +// Resolver implements server discovery, caching well-known lookups. +type Resolver struct { + // HTTPClient fetches /.well-known/matrix/server. It should verify TLS + // normally: the well-known document is fetched from the server name + // itself, over ordinary HTTPS. + HTTPClient *http.Client + + // LookupSRV is net.DefaultResolver.LookupSRV unless a test replaces it. + LookupSRV func(ctx context.Context, service, proto, name string) (string, []*net.SRV, error) + + Now func() time.Time + + mu sync.Mutex + cache map[string]wellKnownEntry +} + +type wellKnownEntry struct { + server string // the delegated m.server, or "" when there is none + expires time.Time +} + +// Well-known cache lifetimes, from the spec's guidance: a successful +// lookup is cached for what Cache-Control says, within these bounds, and a +// failed one for an hour, so that a server with no well-known document is +// not asked for one on every request. +const ( + wellKnownDefaultTTL = 24 * time.Hour + wellKnownMaxTTL = 48 * time.Hour + wellKnownErrorTTL = time.Hour + + // wellKnownMaxBytes caps the document. A real one is a line of JSON. + wellKnownMaxBytes = 64 << 10 +) + +// NewResolver builds a resolver around an HTTP client. +func NewResolver(client *http.Client) *Resolver { + if client == nil { + client = &http.Client{Timeout: 30 * time.Second} + } + return &Resolver{ + HTTPClient: client, + LookupSRV: net.DefaultResolver.LookupSRV, + Now: time.Now, + cache: map[string]wellKnownEntry{}, + } +} + +// Resolve runs the discovery algorithm for a server name. +func (r *Resolver) Resolve(ctx context.Context, serverName string) (Resolved, error) { + name, err := ParseServerName(serverName) + if err != nil { + return Resolved{}, err + } + + // Step 1: an IP literal is its own address. + if name.IsIP { + port := name.Port + if port == 0 { + port = DefaultFederationPort + } + return Resolved{Addresses: []string{hostPort(name.Host, port)}, Host: serverName}, nil + } + // Step 2: an explicit port means no delegation of any kind. + if name.Port != 0 { + return Resolved{Addresses: []string{hostPort(name.Host, name.Port)}, Host: serverName}, nil + } + + // Step 3: a well-known document can delegate to another name. + if delegated := r.wellKnown(ctx, name.Host); delegated != "" { + dname, err := ParseServerName(delegated) + if err == nil { + switch { + case dname.IsIP: + port := dname.Port + if port == 0 { + port = DefaultFederationPort + } + return Resolved{Addresses: []string{hostPort(dname.Host, port)}, Host: delegated}, nil + case dname.Port != 0: + return Resolved{Addresses: []string{hostPort(dname.Host, dname.Port)}, Host: delegated}, nil + default: + return r.srvOrDefault(ctx, dname.Host), nil + } + } + // A malformed delegation is treated as no delegation, which is + // what the spec asks for an invalid well-known response. + } + + // Steps 4 to 6: SRV records, then the default port. + return r.srvOrDefault(ctx, name.Host), nil +} + +// srvOrDefault looks up the SRV records of a hostname, the current service +// name first and the deprecated one second, falling back to port 8448. +// The Host header stays the hostname whatever the SRV target is. +func (r *Resolver) srvOrDefault(ctx context.Context, host string) Resolved { + for _, service := range []string{"matrix-fed", "matrix"} { + _, records, err := r.LookupSRV(ctx, service, "tcp", host) + if err != nil || len(records) == 0 { + continue + } + var addrs []string + for _, rec := range records { + target := strings.TrimSuffix(rec.Target, ".") + if target == "" { + continue + } + addrs = append(addrs, hostPort(target, int(rec.Port))) + } + if len(addrs) > 0 { + return Resolved{Addresses: addrs, Host: host} + } + } + return Resolved{Addresses: []string{hostPort(host, DefaultFederationPort)}, Host: host} +} + +// wellKnown returns the delegated server name for a host, or "". +func (r *Resolver) wellKnown(ctx context.Context, host string) string { + now := r.Now() + r.mu.Lock() + if entry, ok := r.cache[host]; ok && now.Before(entry.expires) { + r.mu.Unlock() + return entry.server + } + r.mu.Unlock() + + server, ttl := r.fetchWellKnown(ctx, host) + + r.mu.Lock() + r.cache[host] = wellKnownEntry{server: server, expires: now.Add(ttl)} + r.mu.Unlock() + return server +} + +// fetchWellKnown fetches and parses the document, returning the +// delegation and how long to believe it. +func (r *Resolver) fetchWellKnown(ctx context.Context, host string) (string, time.Duration) { + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+host+"/.well-known/matrix/server", nil) + if err != nil { + return "", wellKnownErrorTTL + } + resp, err := r.HTTPClient.Do(req) + if err != nil { + return "", wellKnownErrorTTL + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return "", wellKnownErrorTTL + } + body, err := io.ReadAll(io.LimitReader(resp.Body, wellKnownMaxBytes)) + if err != nil { + return "", wellKnownErrorTTL + } + var doc struct { + Server string `json:"m.server"` + } + if err := json.Unmarshal(body, &doc); err != nil || doc.Server == "" { + return "", wellKnownErrorTTL + } + return doc.Server, cacheTTL(resp.Header.Get("Cache-Control")) +} + +// cacheTTL reads max-age from a Cache-Control header, within the bounds +// the spec sets for a well-known lookup. +func cacheTTL(header string) time.Duration { + for _, directive := range strings.Split(header, ",") { + name, value, ok := strings.Cut(strings.TrimSpace(directive), "=") + if !ok || !strings.EqualFold(name, "max-age") { + continue + } + seconds, err := strconv.Atoi(strings.Trim(value, `"`)) + if err != nil || seconds < 0 { + break + } + ttl := time.Duration(seconds) * time.Second + if ttl > wellKnownMaxTTL { + ttl = wellKnownMaxTTL + } + return ttl + } + return wellKnownDefaultTTL +} diff --git a/server/internal/federation/discovery_test.go b/server/internal/federation/discovery_test.go new file mode 100644 index 0000000..adb8566 --- /dev/null +++ b/server/internal/federation/discovery_test.go @@ -0,0 +1,185 @@ +package federation + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestParseServerName(t *testing.T) { + cases := []struct { + in string + want ServerName + bad bool + }{ + {in: "example.org", want: ServerName{Host: "example.org"}}, + {in: "example.org:8448", want: ServerName{Host: "example.org", Port: 8448}}, + {in: "1.2.3.4", want: ServerName{Host: "1.2.3.4", IsIP: true}}, + {in: "1.2.3.4:80", want: ServerName{Host: "1.2.3.4", Port: 80, IsIP: true}}, + {in: "[::1]", want: ServerName{Host: "::1", IsIP: true}}, + {in: "[2001:db8::1]:8448", want: ServerName{Host: "2001:db8::1", Port: 8448, IsIP: true}}, + {in: "", bad: true}, + {in: "::1", bad: true}, + {in: "example.org:", bad: true}, + {in: "example.org:0", bad: true}, + {in: "example.org:70000", bad: true}, + {in: "exa mple.org", bad: true}, + {in: "[1.2.3.4]", bad: true}, + {in: "[::1", bad: true}, + } + for _, tc := range cases { + got, err := ParseServerName(tc.in) + if tc.bad { + if !errors.Is(err, ErrInvalidServerName) { + t.Errorf("ParseServerName(%q) = %+v, %v; want an error", tc.in, got, err) + } + continue + } + if err != nil || got != tc.want { + t.Errorf("ParseServerName(%q) = %+v, %v; want %+v", tc.in, got, err, tc.want) + } + } +} + +// fakeDNS answers SRV lookups from a table. +type fakeDNS map[string][]*net.SRV + +func (f fakeDNS) lookup(_ context.Context, service, proto, name string) (string, []*net.SRV, error) { + records, ok := f["_"+service+"._"+proto+"."+name] + if !ok { + return "", nil, &net.DNSError{Err: "no such host", Name: name, IsNotFound: true} + } + return "", records, nil +} + +// wellKnownServer serves /.well-known/matrix/server for any Host, from a +// table, counting requests. +func wellKnownServer(t *testing.T, docs map[string]string, hits *atomic.Int32) *Resolver { + t.Helper() + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + doc, ok := docs[strings.Split(r.Host, ":")[0]] + if !ok || r.URL.Path != "/.well-known/matrix/server" { + http.NotFound(w, r) + return + } + w.Header().Set("Cache-Control", "max-age=60") + _, _ = w.Write([]byte(doc)) + })) + t.Cleanup(srv.Close) + + // Every hostname is dialled at the test server, whose certificate the + // client trusts, so the well-known fetch is real HTTPS to a fake name. + transport := srv.Client().Transport.(*http.Transport).Clone() + addr := srv.Listener.Addr().String() + transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, addr) + } + transport.TLSClientConfig.InsecureSkipVerify = true //nolint:gosec // test server with a fixed certificate + r := NewResolver(&http.Client{Transport: transport}) + return r +} + +func TestResolve(t *testing.T) { + var hits atomic.Int32 + r := wellKnownServer(t, map[string]string{ + "delegated.example": `{"m.server":"matrix.delegated.example"}`, + "withport.example": `{"m.server":"matrix.withport.example:443"}`, + "toip.example": `{"m.server":"10.0.0.1"}`, + "broken.example": `{"m.server":"not a name"}`, + }, &hits) + r.LookupSRV = fakeDNS{ + "_matrix-fed._tcp.matrix.delegated.example": {{Target: "fed.delegated.example.", Port: 8443}}, + "_matrix-fed._tcp.srv.example": {{Target: "a.srv.example.", Port: 1}, {Target: "b.srv.example.", Port: 2}}, + "_matrix._tcp.legacy.example": {{Target: "old.legacy.example.", Port: 9}}, + }.lookup + + cases := []struct { + name string + addresses []string + host string + }{ + {"1.2.3.4", []string{"1.2.3.4:8448"}, "1.2.3.4"}, + {"[::1]:9000", []string{"[::1]:9000"}, "[::1]:9000"}, + {"explicit.example:1234", []string{"explicit.example:1234"}, "explicit.example:1234"}, + // Delegation by well-known, then SRV on the delegated name: the + // SRV target is dialled, the delegated name is what TLS checks. + {"delegated.example", []string{"fed.delegated.example:8443"}, "matrix.delegated.example"}, + {"withport.example", []string{"matrix.withport.example:443"}, "matrix.withport.example:443"}, + {"toip.example", []string{"10.0.0.1:8448"}, "10.0.0.1"}, + {"broken.example", []string{"broken.example:8448"}, "broken.example"}, + {"srv.example", []string{"a.srv.example:1", "b.srv.example:2"}, "srv.example"}, + {"legacy.example", []string{"old.legacy.example:9"}, "legacy.example"}, + {"plain.example", []string{"plain.example:8448"}, "plain.example"}, + } + for _, tc := range cases { + got, err := r.Resolve(t.Context(), tc.name) + if err != nil { + t.Errorf("Resolve(%s): %v", tc.name, err) + continue + } + if !slices.Equal(got.Addresses, tc.addresses) || got.Host != tc.host { + t.Errorf("Resolve(%s) = %+v, want %v via Host %s", tc.name, got, tc.addresses, tc.host) + } + } + + if got, _ := r.Resolve(t.Context(), "delegated.example"); got.TLSServerName() != "matrix.delegated.example" { + t.Errorf("TLSServerName = %q", got.TLSServerName()) + } + if got, _ := r.Resolve(t.Context(), "[::1]:9000"); got.TLSServerName() != "::1" { + t.Errorf("TLSServerName of an IPv6 literal = %q", got.TLSServerName()) + } +} + +func TestWellKnownIsCached(t *testing.T) { + var hits atomic.Int32 + r := wellKnownServer(t, map[string]string{"cached.example": `{"m.server":"other.example:443"}`}, &hits) + r.LookupSRV = fakeDNS{}.lookup + now := time.Unix(1_700_000_000, 0) + r.Now = func() time.Time { return now } + + for range 3 { + if _, err := r.Resolve(t.Context(), "cached.example"); err != nil { + t.Fatal(err) + } + } + // A server with no document is cached too, as a failure. + for range 3 { + if _, err := r.Resolve(t.Context(), "missing.example"); err != nil { + t.Fatal(err) + } + } + if n := hits.Load(); n != 2 { + t.Errorf("%d well-known fetches, want one per name", n) + } + + now = now.Add(2 * time.Minute) // past max-age=60 + if _, err := r.Resolve(t.Context(), "cached.example"); err != nil { + t.Fatal(err) + } + if n := hits.Load(); n != 3 { + t.Errorf("%d well-known fetches after expiry, want 3", n) + } +} + +func TestCacheTTL(t *testing.T) { + cases := map[string]time.Duration{ + "": wellKnownDefaultTTL, + "max-age=3600": time.Hour, + "public, max-age=120": 2 * time.Minute, + "max-age=999999999": wellKnownMaxTTL, + "max-age=nonsense": wellKnownDefaultTTL, + } + for header, want := range cases { + if got := cacheTTL(header); got != want { + t.Errorf("cacheTTL(%q) = %v, want %v", header, got, want) + } + } +} diff --git a/server/internal/federation/gate.go b/server/internal/federation/gate.go index a2d8479..1fa6126 100644 --- a/server/internal/federation/gate.go +++ b/server/internal/federation/gate.go @@ -1,12 +1,12 @@ -// Package federation holds the server-to-server surface. In Phase 0 that -// is only the gate: the piece that makes federation a runtime mode rather -// than a build-time one. +// Package federation is the Server-Server protocol, with no knowledge of +// rooms: the gate and the allow/deny policy, X-Matrix request signing, +// server discovery, the signed HTTP client and the key ring. // // Every federation route is registered through Gate.Protect, so flipping // federation.enabled through the admin API takes effect on the next request -// without re-registering routes or restarting the process. The transports -// that the switch also has to start and stop (outbound transaction queue, -// key fetcher, remote media) arrive in Phase 5 as federation.Module. +// without re-registering routes or restarting the process. What the switch +// also has to start and stop — the outbound queue — is the federationapi +// package's Module; see docs/federation.md. package federation import ( diff --git a/server/internal/federation/keys.go b/server/internal/federation/keys.go new file mode 100644 index 0000000..56ae8d0 --- /dev/null +++ b/server/internal/federation/keys.go @@ -0,0 +1,463 @@ +package federation + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "errors" + "fmt" + "log/slog" + "sort" + "sync" + "time" + + "github.com/dariuszjaniak/warp/server/internal/matrix/canonicaljson" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Server keys: publishing ours, and fetching, caching and using everybody +// else's. +// +// Every signature another server makes — on an event, on a request — is +// checked against a key that server publishes at /_matrix/key/v2/server. +// The key ring fetches that document, checks that it is signed by the keys +// it lists (a key response vouches for itself; what makes it trustworthy +// is that it came from the server over TLS, or from a notary we chose to +// trust), and caches each key for as long as the server said it would stay +// valid — but never more than a week, which is the spec's cap on how long +// one fetch may be believed. + +// ErrUnknownKey means a server's key could not be found: not cached, and +// neither the server nor any notary produced it. +var ErrUnknownKey = errors.New("signing key not found") + +// maxKeyValidity caps how far ahead a fetched key's valid_until_ts is +// believed. +const maxKeyValidity = 7 * 24 * time.Hour + +// fetchFailureBackoff stops a server whose key server is down from being +// asked again on every request that needs one of its keys. +const fetchFailureBackoff = time.Minute + +// verifyKey and oldVerifyKey are the key objects of a key response. +type verifyKey struct { + Key string `json:"key"` +} + +type oldVerifyKey struct { + Key string `json:"key"` + ExpiredTS int64 `json:"expired_ts"` +} + +// serverKeyResponse is the document at /_matrix/key/v2/server. +type serverKeyResponse struct { + ServerName string `json:"server_name"` + ValidUntilTS int64 `json:"valid_until_ts"` + VerifyKeys map[string]verifyKey `json:"verify_keys"` + OldVerifyKeys map[string]oldVerifyKey `json:"old_verify_keys"` +} + +// KeyRingOptions configure a KeyRing. +type KeyRingOptions struct { + ServerName string + Key *signing.KeyPair + + // Store caches fetched keys across restarts. + Store storage.FederationStore + + // Client fetches keys. It refuses everything while federation is off, + // which is exactly when no remote key can be needed. + Client *Client + + // Notaries are the servers asked for a key the server itself cannot + // be reached for. + Notaries []string + + // Validity is how long our own key response says our key is valid. + Validity time.Duration + + Logger *slog.Logger + Now func() time.Time +} + +// KeyRing holds this server's key and everybody else's. +type KeyRing struct { + serverName string + key *signing.KeyPair + store storage.FederationStore + client *Client + notaries []string + validity time.Duration + log *slog.Logger + now func() time.Time + + mu sync.Mutex + cache map[string]map[string]storage.ServerKey // server -> key ID -> key + failedAt map[string]time.Time + inflight map[string]*fetchCall +} + +type fetchCall struct { + done chan struct{} + err error +} + +// NewKeyRing builds a key ring. +func NewKeyRing(opts KeyRingOptions) *KeyRing { + if opts.Logger == nil { + opts.Logger = slog.Default() + } + if opts.Now == nil { + opts.Now = time.Now + } + if opts.Validity <= 0 { + opts.Validity = maxKeyValidity + } + return &KeyRing{ + serverName: opts.ServerName, + key: opts.Key, + store: opts.Store, + client: opts.Client, + notaries: opts.Notaries, + validity: opts.Validity, + log: opts.Logger, + now: opts.Now, + cache: map[string]map[string]storage.ServerKey{}, + failedAt: map[string]time.Time{}, + inflight: map[string]*fetchCall{}, + } +} + +// OwnKeyResponse is this server's signed key document, as served at +// /_matrix/key/v2/server and handed out by the notary endpoints. +func (k *KeyRing) OwnKeyResponse() (json.RawMessage, error) { + doc := serverKeyResponse{ + ServerName: k.serverName, + ValidUntilTS: k.now().Add(k.validity).UnixMilli(), + VerifyKeys: map[string]verifyKey{k.key.KeyID(): {Key: k.key.PublicBase64()}}, + OldVerifyKeys: map[string]oldVerifyKey{}, + } + raw, err := json.Marshal(doc) + if err != nil { + return nil, err + } + return signing.SignJSON(k.serverName, k.key, raw) +} + +// PublicKey returns a server's key, fetching it when it is not cached or +// is not known to be valid at atTS. +// +// atTS is the time the key has to have been valid at, in milliseconds: +// the event's origin_server_ts when the room version enforces key +// validity, now for a request. Zero accepts any key the server has ever +// published, which is the rule for events in room versions before 5. +func (k *KeyRing) PublicKey(ctx context.Context, serverName, keyID string, atTS int64) (ed25519.PublicKey, error) { + if serverName == k.serverName { + if keyID == k.key.KeyID() { + return k.key.Public, nil + } + return nil, fmt.Errorf("%w: %s is not a key of this server", ErrUnknownKey, keyID) + } + + if key, ok, err := k.cached(ctx, serverName, keyID); err != nil { + return nil, err + } else if ok && usableAt(key, atTS) { + return decodeKey(key) + } + + if err := k.fetch(ctx, serverName); err != nil { + // A key we already have may still be usable for an event whose + // timestamp the old validity covers; only the refresh failed. + if key, ok, _ := k.cached(ctx, serverName, keyID); ok && usableAt(key, atTS) { + return decodeKey(key) + } + return nil, err + } + key, ok, err := k.cached(ctx, serverName, keyID) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("%w: %s has no key %s", ErrUnknownKey, serverName, keyID) + } + if !usableAt(key, atTS) { + return nil, fmt.Errorf("%w: key %s of %s was not valid at %d", ErrUnknownKey, keyID, serverName, atTS) + } + return decodeKey(key) +} + +// usableAt reports whether a key may verify something signed at atTS. +func usableAt(key storage.ServerKey, atTS int64) bool { + if atTS == 0 { + return true + } + if key.ExpiredTS != 0 { + return atTS <= key.ExpiredTS + } + return atTS <= key.ValidUntilTS +} + +func decodeKey(key storage.ServerKey) (ed25519.PublicKey, error) { + raw, err := signing.DecodeBase64(key.PublicKey) + if err != nil || len(raw) != ed25519.PublicKeySize { + return nil, fmt.Errorf("%w: key %s of %s is malformed", ErrUnknownKey, key.KeyID, key.ServerName) + } + return ed25519.PublicKey(raw), nil +} + +// cached reads a key from memory, then from the store. +func (k *KeyRing) cached(ctx context.Context, serverName, keyID string) (storage.ServerKey, bool, error) { + k.mu.Lock() + key, ok := k.cache[serverName][keyID] + k.mu.Unlock() + if ok { + return key, true, nil + } + + keys, err := k.store.ServerKeys(ctx, serverName, []string{keyID}) + if err != nil { + return storage.ServerKey{}, false, err + } + if len(keys) == 0 { + return storage.ServerKey{}, false, nil + } + k.remember(keys) + return keys[0], true, nil +} + +func (k *KeyRing) remember(keys []storage.ServerKey) { + k.mu.Lock() + defer k.mu.Unlock() + for _, key := range keys { + if k.cache[key.ServerName] == nil { + k.cache[key.ServerName] = map[string]storage.ServerKey{} + } + k.cache[key.ServerName][key.KeyID] = key + } +} + +// fetch refreshes a server's keys, once at a time per server. +func (k *KeyRing) fetch(ctx context.Context, serverName string) error { + k.mu.Lock() + if at, failed := k.failedAt[serverName]; failed && k.now().Sub(at) < fetchFailureBackoff { + k.mu.Unlock() + return fmt.Errorf("%w: fetching the keys of %s failed recently", ErrUnknownKey, serverName) + } + if call, ok := k.inflight[serverName]; ok { + k.mu.Unlock() + select { + case <-call.done: + return call.err + case <-ctx.Done(): + return ctx.Err() + } + } + call := &fetchCall{done: make(chan struct{})} + k.inflight[serverName] = call + k.mu.Unlock() + + call.err = k.fetchKeys(ctx, serverName) + + k.mu.Lock() + delete(k.inflight, serverName) + if call.err != nil { + k.failedAt[serverName] = k.now() + } else { + delete(k.failedAt, serverName) + } + k.mu.Unlock() + close(call.done) + return call.err +} + +// fetchKeys asks the server, then each notary in turn. +func (k *KeyRing) fetchKeys(ctx context.Context, serverName string) error { + raw, err := k.client.ServerKeys(ctx, serverName) + if err == nil { + if err = k.acceptResponse(ctx, serverName, raw, ""); err == nil { + return nil + } + } + k.log.Debug("could not fetch keys directly", "server_name", serverName, "error", err) + + for _, notary := range k.notaries { + if notary == serverName { + continue + } + responses, nerr := k.client.QueryKeys(ctx, notary, map[string]map[string]KeyCriteria{serverName: {}}) + if nerr != nil { + k.log.Debug("notary could not supply keys", "notary", notary, "server_name", serverName, "error", nerr) + continue + } + for _, resp := range responses { + if nerr = k.acceptResponse(ctx, serverName, resp, notary); nerr == nil { + return nil + } + } + } + return fmt.Errorf("%w: no keys for %s: %v", ErrUnknownKey, serverName, err) +} + +// acceptResponse checks a key response and stores its keys. +// +// The response has to be about the server asked for, and every signature +// the server made on it has to come from one of the keys it lists and has +// to verify — a response that cites a key it does not publish is either +// broken or forged. A response relayed by a notary must also carry the +// notary's own signature, which is what the notary is trusted for. +func (k *KeyRing) acceptResponse(ctx context.Context, serverName string, raw []byte, notary string) error { + var doc serverKeyResponse + if err := json.Unmarshal(raw, &doc); err != nil { + return fmt.Errorf("decode key response: %w", err) + } + if doc.ServerName != serverName { + return fmt.Errorf("key response is for %q, not %q", doc.ServerName, serverName) + } + sigs, err := signing.ExtractSignatures(raw) + if err != nil { + return err + } + if len(sigs[serverName]) == 0 { + return fmt.Errorf("key response from %s is not signed by it", serverName) + } + for keyID := range sigs[serverName] { + vk, ok := doc.VerifyKeys[keyID] + if !ok { + return fmt.Errorf("key response from %s is signed with %s, which it does not list", serverName, keyID) + } + pub, err := signing.DecodeBase64(vk.Key) + if err != nil || len(pub) != ed25519.PublicKeySize { + return fmt.Errorf("key %s of %s is malformed", keyID, serverName) + } + if err := signing.VerifyJSON(serverName, keyID, pub, raw); err != nil { + return fmt.Errorf("key response from %s: %w", serverName, err) + } + } + if notary != "" { + if err := k.verifyJSONAt(ctx, notary, 0, raw); err != nil { + return fmt.Errorf("key response for %s relayed by %s: %w", serverName, notary, err) + } + } + + now := k.now() + validUntil := min(doc.ValidUntilTS, now.Add(maxKeyValidity).UnixMilli()) + var keys []storage.ServerKey + for keyID, vk := range doc.VerifyKeys { + keys = append(keys, storage.ServerKey{ + ServerName: serverName, KeyID: keyID, PublicKey: vk.Key, + ValidUntilTS: validUntil, FetchedAtMS: now.UnixMilli(), + }) + } + for keyID, ok := range doc.OldVerifyKeys { + keys = append(keys, storage.ServerKey{ + ServerName: serverName, KeyID: keyID, PublicKey: ok.Key, + ValidUntilTS: ok.ExpiredTS, ExpiredTS: ok.ExpiredTS, FetchedAtMS: now.UnixMilli(), + }) + } + sort.Slice(keys, func(i, j int) bool { return keys[i].KeyID < keys[j].KeyID }) + if err := k.store.StoreServerKeys(ctx, keys); err != nil { + return err + } + k.remember(keys) + return nil +} + +// VerifyJSON checks that a signed object carries a valid signature from +// serverName, made with a key valid now. +func (k *KeyRing) VerifyJSON(ctx context.Context, serverName string, msg []byte) error { + return k.verifyJSONAt(ctx, serverName, k.now().UnixMilli(), msg) +} + +// verifyJSONAt is VerifyJSON with the time the key must be valid at. +// +// One good signature is enough. A server rotating its key signs with the +// new one, and an object it signed with both is still its object if we +// can only fetch one of them. +func (k *KeyRing) verifyJSONAt(ctx context.Context, serverName string, atTS int64, msg []byte) error { + sigs, err := signing.ExtractSignatures(msg) + if err != nil { + return err + } + keyIDs := make([]string, 0, len(sigs[serverName])) + for keyID := range sigs[serverName] { + if algorithm, _, ok := signing.SplitKeyID(keyID); ok && algorithm == signing.KeyIDPrefix { + keyIDs = append(keyIDs, keyID) + } + } + if len(keyIDs) == 0 { + return fmt.Errorf("%w: from %s", signing.ErrNoSignature, serverName) + } + sort.Strings(keyIDs) + + var lastErr error + for _, keyID := range keyIDs { + pub, err := k.PublicKey(ctx, serverName, keyID, atTS) + if err != nil { + lastErr = err + continue + } + if err := signing.VerifyJSON(serverName, keyID, pub, msg); err != nil { + lastErr = err + continue + } + return nil + } + return lastErr +} + +// VerifyEventSignature implements auth.Verifier, and is what every event +// from another server is checked with. +func (k *KeyRing) VerifyEventSignature(ctx context.Context, serverName string, e *event.Event) error { + keyIDs := make([]string, 0, len(e.Signatures()[serverName])) + for keyID := range e.Signatures()[serverName] { + if algorithm, _, ok := signing.SplitKeyID(keyID); ok && algorithm == signing.KeyIDPrefix { + keyIDs = append(keyIDs, keyID) + } + } + if len(keyIDs) == 0 { + return fmt.Errorf("%w: event %s is not signed by %s", signing.ErrNoSignature, e.EventID(), serverName) + } + sort.Strings(keyIDs) + + var atTS int64 + if e.Rules().EnforceSigningKeyValidity { + atTS = e.OriginServerTS() + } + var lastErr error + for _, keyID := range keyIDs { + pub, err := k.PublicKey(ctx, serverName, keyID, atTS) + if err != nil { + lastErr = err + continue + } + if err := e.VerifySignature(serverName, keyID, pub); err != nil { + lastErr = err + continue + } + return nil + } + return lastErr +} + +// NotaryResponse answers a /key/v2/query for this server's own keys. The +// key ring does not relay other servers' responses — it keeps their keys, +// not the documents they came in — so a query about anybody else is +// answered with nothing, which a requester treats as "ask somebody else". +func (k *KeyRing) NotaryResponse(serverName string) ([]json.RawMessage, error) { + if serverName != k.serverName { + return nil, nil + } + own, err := k.OwnKeyResponse() + if err != nil { + return nil, err + } + // The response is already signed by this server; as the notary it is + // the same signature, and canonical JSON keeps it byte-identical. + canonical, err := canonicaljson.Encode(own) + if err != nil { + return nil, err + } + return []json.RawMessage{canonical}, nil +} diff --git a/server/internal/federation/keys_test.go b/server/internal/federation/keys_test.go new file mode 100644 index 0000000..e59a23d --- /dev/null +++ b/server/internal/federation/keys_test.go @@ -0,0 +1,259 @@ +package federation + +import ( + "context" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" + "github.com/dariuszjaniak/warp/server/internal/storage" + _ "github.com/dariuszjaniak/warp/server/internal/storage/drivers" +) + +// testStore opens a migrated SQLite database for the key cache. +func testStore(t *testing.T) storage.FederationStore { + t.Helper() + db, err := storage.Open(t.Context(), storage.Config{ + Driver: storage.DriverSQLite, + ConnectionString: "file:" + strings.ReplaceAll(t.TempDir(), `\`, "/") + "/warp.db", + }, nil) + if err != nil { + t.Fatalf("open: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if err := db.Migrate(t.Context()); err != nil { + t.Fatalf("migrate: %v", err) + } + return db.Federation() +} + +// remote is a fake homeserver: a TLS server answering for a server name, +// with its own key. +type remote struct { + name string + key *signing.KeyPair + srv *httptest.Server + mux *http.ServeMux + keyHits atomic.Int32 + + // validFor is the validity its key response claims. + validFor time.Duration +} + +func newRemote(t *testing.T, name string) *remote { + t.Helper() + key, err := signing.Generate("") + if err != nil { + t.Fatal(err) + } + r := &remote{name: name, key: key, mux: http.NewServeMux(), validFor: time.Hour} + r.mux.HandleFunc("GET /_matrix/key/v2/server", func(w http.ResponseWriter, _ *http.Request) { + r.keyHits.Add(1) + doc, _ := json.Marshal(serverKeyResponse{ + ServerName: r.name, + ValidUntilTS: time.Now().Add(r.validFor).UnixMilli(), + VerifyKeys: map[string]verifyKey{r.key.KeyID(): {Key: r.key.PublicBase64()}}, + OldVerifyKeys: map[string]oldVerifyKey{}, + }) + signed, _ := signing.SignJSON(r.name, r.key, doc) + _, _ = w.Write(signed) + }) + r.srv = httptest.NewTLSServer(r.mux) + t.Cleanup(r.srv.Close) + return r +} + +// clientFor builds a client that resolves every remote in the list to its +// test server. The Host is "example.com", the name the test certificate is +// issued for, and the address is the listener: the dial-target path is +// exercised on every request. +func clientFor(t *testing.T, serverName string, key *signing.KeyPair, remotes ...*remote) *Client { + t.Helper() + byName := map[string]*remote{} + for _, r := range remotes { + byName[r.name] = r + } + var tlsConfig = remotes[0].srv.Client().Transport.(*http.Transport).TLSClientConfig.Clone() + return NewClient(ClientOptions{ + ServerName: serverName, + Key: key, + TLSConfig: tlsConfig, + Enabled: func() bool { return true }, + Resolve: func(_ context.Context, name string) (Resolved, error) { + r, ok := byName[name] + if !ok { + return Resolved{}, errors.New("no such test server") + } + _, port, _ := net.SplitHostPort(r.srv.Listener.Addr().String()) + return Resolved{Addresses: []string{r.srv.Listener.Addr().String()}, Host: "example.com:" + port}, nil + }, + }) +} + +func newKeyRing(t *testing.T, name string, remotes ...*remote) (*KeyRing, *signing.KeyPair) { + t.Helper() + key, err := signing.Generate("") + if err != nil { + t.Fatal(err) + } + return NewKeyRing(KeyRingOptions{ + ServerName: name, + Key: key, + Store: testStore(t), + Client: clientFor(t, name, key, remotes...), + }), key +} + +func TestKeyRingFetchesAndCaches(t *testing.T) { + r := newRemote(t, "remote.example") + ring, _ := newKeyRing(t, "warp.example", r) + + pub, err := ring.PublicKey(t.Context(), "remote.example", r.key.KeyID(), time.Now().UnixMilli()) + if err != nil { + t.Fatalf("PublicKey: %v", err) + } + if !pub.Equal(r.key.Public) { + t.Error("fetched the wrong key") + } + for range 3 { + if _, err := ring.PublicKey(t.Context(), "remote.example", r.key.KeyID(), time.Now().UnixMilli()); err != nil { + t.Fatal(err) + } + } + if n := r.keyHits.Load(); n != 1 { + t.Errorf("%d key fetches, want 1", n) + } + + // A key needed for a time past its validity is fetched again, in case + // the server has extended it. + if _, err := ring.PublicKey(t.Context(), "remote.example", r.key.KeyID(), + time.Now().Add(2*time.Hour).UnixMilli()); err == nil { + t.Error("a key was accepted for a time past its validity") + } + if n := r.keyHits.Load(); n != 2 { + t.Errorf("%d key fetches after asking past validity, want 2", n) + } + + // Validity is capped at a week, whatever the server claims. + r.validFor = 365 * 24 * time.Hour + ring2, _ := newKeyRing(t, "warp.example", r) + if _, err := ring2.PublicKey(t.Context(), "remote.example", r.key.KeyID(), + time.Now().Add(30*24*time.Hour).UnixMilli()); err == nil { + t.Error("a key was believed for longer than a week") + } +} + +func TestKeyRingRefusesAForgedResponse(t *testing.T) { + r := newRemote(t, "remote.example") + impostor, _ := signing.Generate("") + r.mux = http.NewServeMux() + r.mux.HandleFunc("GET /_matrix/key/v2/server", func(w http.ResponseWriter, _ *http.Request) { + // Lists the real key but is signed with another one. + doc, _ := json.Marshal(serverKeyResponse{ + ServerName: r.name, + ValidUntilTS: time.Now().Add(time.Hour).UnixMilli(), + VerifyKeys: map[string]verifyKey{r.key.KeyID(): {Key: impostor.PublicBase64()}}, + }) + signed, _ := signing.SignJSON(r.name, r.key, doc) + _, _ = w.Write(signed) + }) + r.srv.Config.Handler = r.mux + + ring, _ := newKeyRing(t, "warp.example", r) + if _, err := ring.PublicKey(t.Context(), "remote.example", r.key.KeyID(), 0); !errors.Is(err, ErrUnknownKey) { + t.Errorf("PublicKey with a forged response = %v, want ErrUnknownKey", err) + } +} + +func TestOwnKeyResponseVerifies(t *testing.T) { + ring, key := newKeyRing(t, "warp.example", newRemote(t, "unused.example")) + doc, err := ring.OwnKeyResponse() + if err != nil { + t.Fatal(err) + } + if err := signing.VerifyJSON("warp.example", key.KeyID(), key.Public, doc); err != nil { + t.Errorf("our own key response does not verify: %v", err) + } + var parsed serverKeyResponse + if err := json.Unmarshal(doc, &parsed); err != nil { + t.Fatal(err) + } + if parsed.VerifyKeys[key.KeyID()].Key != key.PublicBase64() { + t.Errorf("verify_keys = %+v", parsed.VerifyKeys) + } +} + +func TestSignedRequestIsAuthenticatedEndToEnd(t *testing.T) { + // The remote here is the *sender*: a key server for remote.example, + // and a client signing as remote.example to warp.example. + sender := newRemote(t, "remote.example") + receiver := newRemote(t, "warp.example") + ring := NewKeyRing(KeyRingOptions{ + ServerName: "warp.example", + Key: receiver.key, + Store: testStore(t), + Client: clientFor(t, "warp.example", receiver.key, sender), + }) + + var gotOrigin atomic.Value + receiver.mux.HandleFunc("PUT /_matrix/federation/v1/send/{txn}", func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + origin, err := ring.AuthenticateRequest(r.Context(), r, body) + if err != nil { + http.Error(w, `{"errcode":"M_UNAUTHORIZED","error":"`+err.Error()+`"}`, http.StatusUnauthorized) + return + } + gotOrigin.Store(origin) + _, _ = w.Write([]byte(`{"pdus":{}}`)) + }) + + client := clientFor(t, "remote.example", sender.key, receiver) + if _, err := client.SendTransaction(t.Context(), "warp.example", "txn/1 with spaces", Transaction{ + Origin: "remote.example", OriginServerTS: 1, PDUs: []json.RawMessage{}, + }); err != nil { + t.Fatalf("SendTransaction: %v", err) + } + if got, _ := gotOrigin.Load().(string); got != "remote.example" { + t.Errorf("authenticated origin = %q", got) + } + + // A request signed by a key the origin does not publish is refused. + stranger, _ := signing.Generate("") + forged := clientFor(t, "remote.example", stranger, receiver) + _, err := forged.SendTransaction(t.Context(), "warp.example", "2", Transaction{Origin: "remote.example"}) + var httpErr *HTTPError + if !errors.As(err, &httpErr) || httpErr.Status != http.StatusUnauthorized { + t.Errorf("forged request = %v, want 401", err) + } +} + +func TestClientRefusesWhileDisabledOrDenied(t *testing.T) { + r := newRemote(t, "remote.example") + key, _ := signing.Generate("") + enabled := false + client := clientFor(t, "warp.example", key, r) + client.enabled = func() bool { return enabled } + + if _, err := client.Version(t.Context(), "remote.example"); !errors.Is(err, ErrDisabled) { + t.Errorf("Version while disabled = %v, want ErrDisabled", err) + } + enabled = true + client.policy = NewPolicy(nil, []string{"remote.example"}) + if _, err := client.Version(t.Context(), "remote.example"); !errors.Is(err, ErrNotAllowed) { + t.Errorf("Version to a denied server = %v, want ErrNotAllowed", err) + } + if _, err := client.Version(t.Context(), "warp.example"); !errors.Is(err, ErrSelf) { + t.Errorf("Version to ourselves = %v, want ErrSelf", err) + } + if r.keyHits.Load() != 0 { + t.Error("a refused request reached the network") + } +} diff --git a/server/internal/federation/requestauth.go b/server/internal/federation/requestauth.go new file mode 100644 index 0000000..d18e56b --- /dev/null +++ b/server/internal/federation/requestauth.go @@ -0,0 +1,68 @@ +package federation + +import ( + "context" + "fmt" + "net/http" +) + +// AuthenticateRequest checks the X-Matrix signature on an inbound request +// and returns the server it came from. +// +// content is the request body, already read; the signature covers it, so +// the caller reads it once and hands the same bytes here and to its own +// decoder. A request may carry one Authorization header per key it is +// signed with, and any one that verifies is enough. +func (k *KeyRing) AuthenticateRequest(ctx context.Context, r *http.Request, content []byte) (string, error) { + headers := r.Header.Values("Authorization") + if len(headers) == 0 { + return "", fmt.Errorf("%w: no Authorization header", ErrBadAuthorization) + } + + var ( + origin string + lastErr error + ) + for _, h := range headers { + x, err := ParseXMatrix(h) + if err != nil { + lastErr = err + continue + } + if origin != "" && x.Origin != origin { + // Two headers naming two origins is one request claiming to + // be from two servers; neither claim is worth checking. + return "", fmt.Errorf("%w: headers name both %s and %s", ErrBadAuthorization, origin, x.Origin) + } + origin = x.Origin + + if x.Destination != "" && x.Destination != k.serverName { + lastErr = fmt.Errorf("%w: addressed to %s, not to this server", ErrBadAuthorization, x.Destination) + continue + } + if x.Origin == k.serverName { + lastErr = fmt.Errorf("%w: a request claiming to be from this server", ErrBadAuthorization) + continue + } + + // The destination in the signed object is whatever the sender put + // in the header, or this server's name when it put nothing: a + // sender old enough to leave it out signed with it absent from + // the header, not from the object. + destination := x.Destination + if destination == "" { + destination = k.serverName + } + signed, err := requestSignedJSON(x, destination, r.Method, r.RequestURI, content) + if err != nil { + lastErr = err + continue + } + if err := k.VerifyJSON(ctx, x.Origin, signed); err != nil { + lastErr = fmt.Errorf("%w: %v", ErrBadAuthorization, err) + continue + } + return x.Origin, nil + } + return "", lastErr +} diff --git a/server/internal/federation/types.go b/server/internal/federation/types.go new file mode 100644 index 0000000..9b1f011 --- /dev/null +++ b/server/internal/federation/types.go @@ -0,0 +1,151 @@ +package federation + +import "encoding/json" + +// The Server-Server API's request and response bodies, shared by the +// client that sends them and the handlers that answer them. +// +// PDUs travel as json.RawMessage throughout. Decoding one needs its room +// version, and only the roomserver knows that; anything that parsed a PDU +// on the way through here would have to guess. + +// Transaction is the body of PUT /send/{txnId}, and the shape /event and +// /backfill answer with. +type Transaction struct { + Origin string `json:"origin"` + OriginServerTS int64 `json:"origin_server_ts"` + PDUs []json.RawMessage `json:"pdus"` + EDUs []EDU `json:"edus,omitempty"` +} + +// EDU is an ephemeral data unit: typing, receipts, presence, to-device. +type EDU struct { + Type string `json:"edu_type"` + Content json.RawMessage `json:"content"` +} + +// The EDU types Warp sends and receives. +const ( + EDUTyping = "m.typing" + EDUReceipt = "m.receipt" + EDUDirectToDev = "m.direct_to_device" + EDUPresence = "m.presence" + EDUDeviceListUp = "m.device_list_update" +) + +// SendTransactionResponse is the answer to PUT /send: a result per PDU. +type SendTransactionResponse struct { + PDUs map[string]PDUResult `json:"pdus"` +} + +// PDUResult is the outcome for one PDU. An empty Error means it was +// accepted. +type PDUResult struct { + Error string `json:"error,omitempty"` +} + +// MakeJoinResponse is the answer to GET /make_join (and /make_leave): an +// unsigned event template the joining server fills in and signs. +type MakeJoinResponse struct { + RoomVersion string `json:"room_version,omitempty"` + Event json.RawMessage `json:"event"` +} + +// SendJoinResponse is the v2 answer to PUT /send_join: the room as of the +// join. +type SendJoinResponse struct { + Origin string `json:"origin"` + State []json.RawMessage `json:"state"` + AuthChain []json.RawMessage `json:"auth_chain"` + + // Event is the join as the resident server signed it, for a join + // under the restricted join rule that it had to countersign. + Event json.RawMessage `json:"event,omitempty"` + + MembersOmitted bool `json:"members_omitted,omitempty"` + ServersInRoom []string `json:"servers_in_room,omitempty"` +} + +// InviteRequest is the v2 body of PUT /invite. +type InviteRequest struct { + RoomVersion string `json:"room_version"` + Event json.RawMessage `json:"event"` + InviteRoomState []json.RawMessage `json:"invite_room_state,omitempty"` +} + +// InviteResponse is the answer to PUT /invite: the invite, countersigned. +type InviteResponse struct { + Event json.RawMessage `json:"event"` +} + +// StateIDsResponse is the answer to GET /state_ids. +type StateIDsResponse struct { + PDUIDs []string `json:"pdu_ids"` + AuthChainIDs []string `json:"auth_chain_ids"` +} + +// StateResponse is the answer to GET /state. +type StateResponse struct { + PDUs []json.RawMessage `json:"pdus"` + AuthChain []json.RawMessage `json:"auth_chain"` +} + +// EventAuthResponse is the answer to GET /event_auth. +type EventAuthResponse struct { + AuthChain []json.RawMessage `json:"auth_chain"` +} + +// MissingEventsRequest is the body of POST /get_missing_events. +type MissingEventsRequest struct { + EarliestEvents []string `json:"earliest_events"` + LatestEvents []string `json:"latest_events"` + Limit int `json:"limit,omitempty"` + MinDepth int64 `json:"min_depth,omitempty"` +} + +// MissingEventsResponse is the answer to POST /get_missing_events. +type MissingEventsResponse struct { + Events []json.RawMessage `json:"events"` +} + +// DirectoryResponse is the answer to GET /query/directory. +type DirectoryResponse struct { + RoomID string `json:"room_id"` + Servers []string `json:"servers"` +} + +// VersionResponse is the answer to GET /version. +type VersionResponse struct { + Server struct { + Name string `json:"name"` + Version string `json:"version"` + } `json:"server"` +} + +// TypingContent is the content of an m.typing EDU. +type TypingContent struct { + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + Typing bool `json:"typing"` +} + +// ReceiptContent is the content of an m.receipt EDU: room ID, then receipt +// type, then user ID. +type ReceiptContent map[string]map[string]map[string]ReceiptData + +// ReceiptData is one user's receipt in an m.receipt EDU. +type ReceiptData struct { + EventIDs []string `json:"event_ids"` + Data struct { + TS int64 `json:"ts"` + ThreadID string `json:"thread_id,omitempty"` + } `json:"data"` +} + +// DirectToDeviceContent is the content of an m.direct_to_device EDU. +type DirectToDeviceContent struct { + Sender string `json:"sender"` + Type string `json:"type"` + MessageID string `json:"message_id"` + Messages map[string]map[string]json.RawMessage `json:"messages"` +} diff --git a/server/internal/federation/xmatrix.go b/server/internal/federation/xmatrix.go new file mode 100644 index 0000000..2fe87d7 --- /dev/null +++ b/server/internal/federation/xmatrix.go @@ -0,0 +1,239 @@ +package federation + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/dariuszjaniak/warp/server/internal/matrix/canonicaljson" + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" +) + +// The X-Matrix authorization scheme: how one server proves to another that +// a request came from it. +// +// The sender signs a small JSON object naming the request — method, URI, +// origin, destination and, when there is one, the JSON body — and sends +// the signature in the Authorization header: +// +// Authorization: X-Matrix origin="a.example",destination="b.example",key="ed25519:k",sig="..." +// +// The receiver rebuilds the same object from the request it actually got +// and checks the signature against the origin's published key. Nothing in +// the header is trusted until that check passes: the origin it names is a +// claim, and it is exactly the claim being checked. + +// ErrBadAuthorization is the root of every X-Matrix parsing and +// verification failure. Each one is answered with M_UNAUTHORIZED. +var ErrBadAuthorization = errors.New("invalid X-Matrix authorization") + +// XMatrix is one parsed X-Matrix header. +type XMatrix struct { + Origin string + + // Destination is the server the request was addressed to. Older + // servers leave it out, and a receiver then has to assume it was + // itself. + Destination string + + // Key is the full key ID, algorithm included. + Key string + + // Sig is the unpadded base64 signature. + Sig string +} + +// maxAuthorizationLength bounds the header before anything is parsed. A +// real one is a couple of hundred bytes; this leaves room for long server +// names without letting a client make the parser chew on a megabyte. +const maxAuthorizationLength = 4096 + +// ParseXMatrix parses an Authorization header value. +// +// The grammar is the spec's: the scheme, then comma-separated key=value +// parameters whose values may be quoted. Quoted values may contain +// backslash escapes, and unknown parameters are ignored, so that a header +// carrying something a newer server added still parses. +func ParseXMatrix(header string) (XMatrix, error) { + if len(header) > maxAuthorizationLength { + return XMatrix{}, fmt.Errorf("%w: header is %d bytes", ErrBadAuthorization, len(header)) + } + scheme, rest, ok := strings.Cut(strings.TrimSpace(header), " ") + if !ok || !strings.EqualFold(scheme, "X-Matrix") { + return XMatrix{}, fmt.Errorf("%w: not an X-Matrix header", ErrBadAuthorization) + } + + var x XMatrix + params, err := parseAuthParams(rest) + if err != nil { + return XMatrix{}, err + } + for name, value := range params { + switch strings.ToLower(name) { + case "origin": + x.Origin = value + case "destination": + x.Destination = value + case "key": + x.Key = value + case "sig": + x.Sig = value + } + } + + switch { + case x.Origin == "": + return XMatrix{}, fmt.Errorf("%w: no origin", ErrBadAuthorization) + case x.Key == "": + return XMatrix{}, fmt.Errorf("%w: no key", ErrBadAuthorization) + case x.Sig == "": + return XMatrix{}, fmt.Errorf("%w: no sig", ErrBadAuthorization) + } + if _, _, ok := signing.SplitKeyID(x.Key); !ok { + return XMatrix{}, fmt.Errorf("%w: key %q is not algorithm:id", ErrBadAuthorization, x.Key) + } + return x, nil +} + +// parseAuthParams splits `a="x",b=y` into its parameters. +func parseAuthParams(s string) (map[string]string, error) { + params := map[string]string{} + i := 0 + for { + // Skip separators and whitespace between parameters. + for i < len(s) && (s[i] == ',' || s[i] == ' ' || s[i] == '\t') { + i++ + } + if i >= len(s) { + return params, nil + } + + eq := strings.IndexByte(s[i:], '=') + if eq <= 0 { + return nil, fmt.Errorf("%w: parameter without a value", ErrBadAuthorization) + } + name := strings.TrimSpace(s[i : i+eq]) + if name == "" || strings.ContainsAny(name, " \t\",") { + return nil, fmt.Errorf("%w: malformed parameter name %q", ErrBadAuthorization, name) + } + i += eq + 1 + + var value string + if i < len(s) && s[i] == '"' { + var b strings.Builder + i++ + closed := false + for i < len(s) { + c := s[i] + if c == '\\' && i+1 < len(s) { + b.WriteByte(s[i+1]) + i += 2 + continue + } + if c == '"' { + closed = true + i++ + break + } + b.WriteByte(c) + i++ + } + if !closed { + return nil, fmt.Errorf("%w: unterminated quoted value", ErrBadAuthorization) + } + value = b.String() + } else { + end := strings.IndexByte(s[i:], ',') + if end < 0 { + end = len(s) - i + } + value = strings.TrimSpace(s[i : i+end]) + i += end + } + + if _, dup := params[name]; dup { + // Two origins in one header is not something to resolve by + // picking one: whichever we picked, the other server meant + // something else. + return nil, fmt.Errorf("%w: parameter %q given twice", ErrBadAuthorization, name) + } + params[name] = value + } +} + +// String renders the header value. Every parameter is quoted, which the +// spec allows and which every implementation accepts. +func (x XMatrix) String() string { + quote := func(s string) string { + return `"` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(s) + `"` + } + return "X-Matrix origin=" + quote(x.Origin) + + ",destination=" + quote(x.Destination) + + ",key=" + quote(x.Key) + + ",sig=" + quote(x.Sig) +} + +// requestJSON is the object an X-Matrix signature covers. +type requestJSON struct { + Method string `json:"method"` + URI string `json:"uri"` + Origin string `json:"origin"` + Destination string `json:"destination"` + Content json.RawMessage `json:"content,omitempty"` +} + +// signableRequest builds the bytes a request signature covers. +// +// uri is the request target exactly as it was sent — path and query, still +// escaped. Both ends must agree on it byte for byte, which is why the +// receiver reads it from the raw request line rather than reassembling it +// from a parsed URL. +func signableRequest(method, uri, origin, destination string, content []byte) ([]byte, error) { + obj := requestJSON{Method: method, URI: uri, Origin: origin, Destination: destination} + if len(content) > 0 { + obj.Content = content + } + raw, err := json.Marshal(obj) + if err != nil { + return nil, fmt.Errorf("encode request for signing: %w", err) + } + return canonicaljson.Encode(raw) +} + +// SignRequest produces the X-Matrix header for an outbound request. +func SignRequest(key *signing.KeyPair, origin, destination, method, uri string, content []byte) (XMatrix, error) { + msg, err := signableRequest(method, uri, origin, destination, content) + if err != nil { + return XMatrix{}, err + } + signed, err := signing.SignJSON(origin, key, msg) + if err != nil { + return XMatrix{}, err + } + sigs, err := signing.ExtractSignatures(signed) + if err != nil { + return XMatrix{}, err + } + sig, _ := sigs.Get(origin, key.KeyID()) + return XMatrix{Origin: origin, Destination: destination, Key: key.KeyID(), Sig: sig}, nil +} + +// requestSignedJSON rebuilds the signed object an inbound header claims to +// cover, with the header's signature attached, ready for VerifyJSON. +func requestSignedJSON(x XMatrix, destination, method, uri string, content []byte) ([]byte, error) { + msg, err := signableRequest(method, uri, x.Origin, destination, content) + if err != nil { + return nil, err + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(msg, &obj); err != nil { + return nil, err + } + sigs, err := json.Marshal(signing.Signatures{x.Origin: {x.Key: x.Sig}}) + if err != nil { + return nil, err + } + obj["signatures"] = sigs + return json.Marshal(obj) +} diff --git a/server/internal/federation/xmatrix_test.go b/server/internal/federation/xmatrix_test.go new file mode 100644 index 0000000..5d9b817 --- /dev/null +++ b/server/internal/federation/xmatrix_test.go @@ -0,0 +1,122 @@ +package federation + +import ( + "errors" + "strings" + "testing" + + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" +) + +func TestParseXMatrix(t *testing.T) { + cases := []struct { + name string + header string + want XMatrix + bad bool + }{ + { + name: "quoted", + header: `X-Matrix origin="a.example",destination="b.example",key="ed25519:k1",sig="c2ln"`, + want: XMatrix{Origin: "a.example", Destination: "b.example", Key: "ed25519:k1", Sig: "c2ln"}, + }, + { + name: "unquoted and spaced", + header: `X-Matrix origin=a.example, key=ed25519:k1 ,sig=c2ln`, + want: XMatrix{Origin: "a.example", Key: "ed25519:k1", Sig: "c2ln"}, + }, + { + name: "escapes and an unknown parameter", + header: `x-matrix origin="a\"b",future="x,y",key="ed25519:k",sig="s"`, + want: XMatrix{Origin: `a"b`, Key: "ed25519:k", Sig: "s"}, + }, + {name: "wrong scheme", header: `Bearer abc`, bad: true}, + {name: "no origin", header: `X-Matrix key="ed25519:k",sig="s"`, bad: true}, + {name: "no sig", header: `X-Matrix origin="a",key="ed25519:k"`, bad: true}, + {name: "key without an algorithm", header: `X-Matrix origin="a",key="k",sig="s"`, bad: true}, + {name: "unterminated quote", header: `X-Matrix origin="a,key="ed25519:k",sig="s"`, bad: true}, + {name: "duplicate origin", header: `X-Matrix origin="a",origin="b",key="ed25519:k",sig="s"`, bad: true}, + {name: "too long", header: "X-Matrix origin=" + strings.Repeat("a", 5000) + `,key="ed25519:k",sig="s"`, bad: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := ParseXMatrix(tc.header) + if tc.bad { + if !errors.Is(err, ErrBadAuthorization) { + t.Fatalf("ParseXMatrix(%q) = %+v, %v; want ErrBadAuthorization", tc.header, got, err) + } + return + } + if err != nil { + t.Fatalf("ParseXMatrix(%q): %v", tc.header, err) + } + if got != tc.want { + t.Errorf("ParseXMatrix(%q) = %+v, want %+v", tc.header, got, tc.want) + } + }) + } +} + +func TestXMatrixStringRoundTrips(t *testing.T) { + x := XMatrix{Origin: `we"ird`, Destination: `b\c`, Key: "ed25519:k", Sig: "c2ln"} + got, err := ParseXMatrix(x.String()) + if err != nil { + t.Fatalf("ParseXMatrix(%q): %v", x.String(), err) + } + if got != x { + t.Errorf("round trip = %+v, want %+v", got, x) + } +} + +func TestSignedRequestVerifies(t *testing.T) { + key, err := signing.Generate("k1") + if err != nil { + t.Fatal(err) + } + content := []byte(`{"b":2,"a":1}`) + x, err := SignRequest(key, "a.example", "b.example", "PUT", "/_matrix/federation/v1/send/1?x=%21", content) + if err != nil { + t.Fatalf("SignRequest: %v", err) + } + + verify := func(method, uri string, body []byte) error { + signed, err := requestSignedJSON(x, "b.example", method, uri, body) + if err != nil { + return err + } + return signing.VerifyJSON("a.example", x.Key, key.Public, signed) + } + if err := verify("PUT", "/_matrix/federation/v1/send/1?x=%21", []byte(`{"a":1,"b":2}`)); err != nil { + t.Errorf("the request as signed does not verify: %v", err) + } + // Each part of the request is covered. + if err := verify("POST", "/_matrix/federation/v1/send/1?x=%21", content); err == nil { + t.Error("a different method verified") + } + if err := verify("PUT", "/_matrix/federation/v1/send/1?x=!", content); err == nil { + t.Error("a differently escaped URI verified") + } + if err := verify("PUT", "/_matrix/federation/v1/send/1?x=%21", []byte(`{"a":1}`)); err == nil { + t.Error("a different body verified") + } +} + +func FuzzParseXMatrix(f *testing.F) { + f.Add(`X-Matrix origin="a.example",destination="b.example",key="ed25519:k1",sig="c2ln"`) + f.Add(`X-Matrix origin=a,key=ed25519:k,sig=s`) + f.Add(`X-Matrix origin="\\",key="ed25519:k",sig="\"`) + f.Fuzz(func(t *testing.T, header string) { + x, err := ParseXMatrix(header) + if err != nil { + return + } + // Whatever parses must survive being written back out. + again, err := ParseXMatrix(x.String()) + if err != nil { + t.Fatalf("re-parsing %q (from %q): %v", x.String(), header, err) + } + if again != x { + t.Fatalf("round trip of %q changed %+v into %+v", header, x, again) + } + }) +} diff --git a/server/internal/federationapi/edus.go b/server/internal/federationapi/edus.go new file mode 100644 index 0000000..ed2c5d8 --- /dev/null +++ b/server/internal/federationapi/edus.go @@ -0,0 +1,98 @@ +package federationapi + +import ( + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "time" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" +) + +// EDUs turns the sync API's local typing, receipts and to-device messages +// into queued EDUs. It implements syncapi.Outbound. +type EDUs struct { + queue *Queue + rooms RoomServers +} + +// NewEDUs builds the adapter. +func NewEDUs(queue *Queue, rooms RoomServers) *EDUs { + return &EDUs{queue: queue, rooms: rooms} +} + +// roomServers is everybody in a room but this server; the queue drops +// this server anyway, and an error is logged rather than returned, because +// a typing notification that does not go out is not worth failing the +// client's request over. +func (e *EDUs) roomServers(ctx context.Context, roomID string) []string { + servers, err := e.rooms.JoinedServers(ctx, roomID) + if err != nil { + e.queue.log.Warn("could not read a room's servers for an EDU", "room_id", roomID, "error", err) + return nil + } + return servers +} + +// Typing implements syncapi.Outbound. +func (e *EDUs) Typing(ctx context.Context, roomID, userID string, typing bool, _ time.Duration) { + if !e.queue.Running() { + return + } + content, _ := json.Marshal(federation.TypingContent{RoomID: roomID, UserID: userID, Typing: typing}) + e.queue.QueueEDU(ctx, e.roomServers(ctx, roomID), federation.EDU{Type: federation.EDUTyping, Content: content}) +} + +// Receipt implements syncapi.Outbound. +func (e *EDUs) Receipt(ctx context.Context, roomID, userID, receiptType, eventID, threadID string, ts int64) { + if !e.queue.Running() || receiptType != "m.read" { + return + } + var data federation.ReceiptData + data.EventIDs = []string{eventID} + data.Data.TS = ts + data.Data.ThreadID = threadID + content, _ := json.Marshal(federation.ReceiptContent{roomID: {receiptType: {userID: data}}}) + e.queue.QueueEDU(ctx, e.roomServers(ctx, roomID), federation.EDU{Type: federation.EDUReceipt, Content: content}) +} + +// ToDevice implements syncapi.Outbound: one m.direct_to_device per +// destination, carrying that server's recipients only. +func (e *EDUs) ToDevice(ctx context.Context, sender, eventType string, messages map[string]map[string]json.RawMessage) { + if !e.queue.Running() { + return + } + byServer := map[string]map[string]map[string]json.RawMessage{} + for userID, devices := range messages { + server, err := id.ServerNameOf(userID) + if err != nil { + continue + } + if byServer[server] == nil { + byServer[server] = map[string]map[string]json.RawMessage{} + } + byServer[server][userID] = devices + } + for server, recipients := range byServer { + content, err := json.Marshal(federation.DirectToDeviceContent{ + Sender: sender, + Type: eventType, + MessageID: messageID(), + Messages: recipients, + }) + if err != nil { + continue + } + e.queue.QueueEDU(ctx, []string{server}, federation.EDU{Type: federation.EDUDirectToDev, Content: content}) + } +} + +// messageID is the per-message identifier a receiving server deduplicates +// a to-device message by. +func messageID() string { + buf := make([]byte, 12) + _, _ = rand.Read(buf) + return base64.RawURLEncoding.EncodeToString(buf) +} diff --git a/server/internal/federationapi/module.go b/server/internal/federationapi/module.go new file mode 100644 index 0000000..6e4c924 --- /dev/null +++ b/server/internal/federationapi/module.go @@ -0,0 +1,96 @@ +// Package federationapi is the Server-Server API as a running component: +// the inbound endpoints other servers call, the outbound queue that sends +// them this server's events, and the Module that starts and stops it all +// with the federation switch. +// +// The protocol itself — X-Matrix signing, server discovery, the HTTP +// client, keys — is in package federation. This package is where that +// protocol meets the roomserver and the sync API, which is why it is +// separate: the roomserver uses the protocol, and this package uses the +// roomserver. +package federationapi + +import ( + "context" + "log/slog" + "sync" +) + +// Module is the switchable part of federation: everything that runs in the +// background and talks to other servers on its own initiative. +// +// The inbound endpoints are not in it. They are behind the gate, which +// reads the switch on every request, so they need nothing started or +// stopped. What does is the outbound queue — the only thing in Warp that +// opens connections to other servers unprompted — and turning federation +// off stops it mid-flight, leaving what it had not sent in the database for +// when federation comes back on. +// +// Requests a component makes on a client's behalf — a join, a key fetch, +// a profile lookup — need nothing stopping either: the client refuses +// every request while the switch is off. +type Module struct { + queue *Queue + log *slog.Logger + + mu sync.Mutex + running bool +} + +// NewModule builds a stopped module. +func NewModule(queue *Queue, log *slog.Logger) *Module { + if log == nil { + log = slog.Default() + } + return &Module{queue: queue, log: log} +} + +// Start brings federation up. Starting a running module does nothing. +func (m *Module) Start(ctx context.Context) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.running { + return nil + } + if err := m.queue.Start(ctx); err != nil { + m.queue.Stop() + return err + } + m.running = true + m.log.Info("federation started") + return nil +} + +// Stop takes federation down and waits for in-flight sends to finish or +// be cancelled. Stopping a stopped module does nothing. +func (m *Module) Stop() { + m.mu.Lock() + defer m.mu.Unlock() + if !m.running { + return + } + m.queue.Stop() + m.running = false + m.log.Info("federation stopped") +} + +// Apply starts or stops the module to match the switch. +func (m *Module) Apply(ctx context.Context, enabled bool) { + if enabled { + if err := m.Start(ctx); err != nil { + m.log.Error("could not start federation", "error", err) + } + return + } + m.Stop() +} + +// Running reports whether the module is up. +func (m *Module) Running() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.running +} + +// Queue exposes the outbound queue, for the admin API. +func (m *Module) Queue() *Queue { return m.queue } diff --git a/server/internal/federationapi/queue.go b/server/internal/federationapi/queue.go new file mode 100644 index 0000000..0c3c79d --- /dev/null +++ b/server/internal/federationapi/queue.go @@ -0,0 +1,563 @@ +package federationapi + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "math/rand/v2" + "slices" + "strconv" + "sync" + "time" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// The outbound queue: every event and EDU this server owes another one. +// +// It has three moving parts. The roomserver hands it every persisted event +// through EventPersisted, which only appends to an in-memory list — it is +// called with the room's write lock held and must not touch the database. +// A dispatcher drains that list, works out each event's destinations and +// writes one queue row per destination. And one worker per destination +// with something queued sends it, in transactions of up to 50 PDUs and +// 100 EDUs, backing off exponentially when the destination fails. +// +// The rows are deleted only once the destination has answered, so a crash +// or a restart loses nothing that had reached the database, and a +// destination that is down gets everything it missed when it comes back. +// What it does lose is the dispatcher's in-memory list: the events +// persisted in the instant before a crash. A remote server notices those +// as gaps the next time it hears from this one, and fetches them. + +// Transaction limits, from the spec. +const ( + maxPDUsPerTxn = 50 + maxEDUsPerTxn = 100 +) + +// Backoff bounds. The first retry comes quickly, because most failures are +// a restart on the other side; the cap is long, because a server that has +// been gone for a day is not coming back in the next minute, and every +// retry costs a connection attempt. +const ( + defaultMinBackoff = 10 * time.Second + defaultMaxBackoff = 6 * time.Hour +) + +// sendTimeout bounds one transaction. +const sendTimeout = 2 * time.Minute + +// Sender is the part of the federation client the queue uses. +type Sender interface { + SendTransaction(ctx context.Context, destination, txnID string, txn federation.Transaction) (federation.SendTransactionResponse, error) +} + +// RoomServers says which servers are in a room. +type RoomServers interface { + JoinedServers(ctx context.Context, roomID string) ([]string, error) +} + +// QueueOptions configure a Queue. +type QueueOptions struct { + ServerName string + Store storage.FederationStore + Sender Sender + Rooms RoomServers + Policy *federation.Policy + + MinBackoff time.Duration + MaxBackoff time.Duration + + Logger *slog.Logger + Now func() time.Time +} + +// Queue is the outbound queue. +type Queue struct { + serverName string + store storage.FederationStore + sender Sender + rooms RoomServers + policy *federation.Policy + minBackoff time.Duration + maxBackoff time.Duration + log *slog.Logger + now func() time.Time + + // txnPrefix makes transaction IDs unique across restarts: a remote + // server deduplicates by (origin, txn ID), and an ID reused with + // different contents would be answered from its cache and the events + // in it never seen. + txnPrefix string + + mu sync.Mutex + running bool + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + incoming []roomserver.Notification + wake chan struct{} + workers map[string]*worker + txnSeq uint64 +} + +// worker sends one destination's queue. +type worker struct { + destination string + wake chan struct{} +} + +// NewQueue builds a stopped queue. +func NewQueue(opts QueueOptions) *Queue { + if opts.Logger == nil { + opts.Logger = slog.Default() + } + if opts.Now == nil { + opts.Now = time.Now + } + if opts.MinBackoff <= 0 { + opts.MinBackoff = defaultMinBackoff + } + if opts.MaxBackoff < opts.MinBackoff { + opts.MaxBackoff = max(defaultMaxBackoff, opts.MinBackoff) + } + if opts.Policy == nil { + opts.Policy = federation.NewPolicy(nil, nil) + } + return &Queue{ + serverName: opts.ServerName, + store: opts.Store, + sender: opts.Sender, + rooms: opts.Rooms, + policy: opts.Policy, + minBackoff: opts.MinBackoff, + maxBackoff: opts.MaxBackoff, + log: opts.Logger, + now: opts.Now, + wake: make(chan struct{}, 1), + workers: map[string]*worker{}, + } +} + +// Start begins sending: the dispatcher, and a worker for every destination +// with something left in the queue from before. +func (q *Queue) Start(ctx context.Context) error { + q.mu.Lock() + if q.running { + q.mu.Unlock() + return nil + } + q.ctx, q.cancel = context.WithCancel(context.WithoutCancel(ctx)) + q.running = true + q.txnPrefix = strconv.FormatInt(q.now().UnixNano(), 36) + q.mu.Unlock() + + q.wg.Add(1) + go q.dispatch() + + destinations, err := q.store.QueuedDestinations(ctx) + if err != nil { + return fmt.Errorf("read the outbound queue: %w", err) + } + for _, d := range destinations { + q.kick(d) + } + q.log.Info("federation queue started", "destinations_with_backlog", len(destinations)) + return nil +} + +// Stop halts every send and waits for the workers to finish. What is +// queued stays queued. +func (q *Queue) Stop() { + q.mu.Lock() + if !q.running { + q.mu.Unlock() + return + } + q.running = false + q.cancel() + q.incoming = nil + q.mu.Unlock() + + q.wg.Wait() + + q.mu.Lock() + q.workers = map[string]*worker{} + q.mu.Unlock() + q.log.Info("federation queue stopped") +} + +// Running reports whether the queue is sending. +func (q *Queue) Running() bool { + q.mu.Lock() + defer q.mu.Unlock() + return q.running +} + +// EventPersisted implements roomserver.Notifier. It must not block, and +// does not: it appends and signals. +// +// While the queue is stopped events are dropped rather than held. That is +// what the federation switch promises — no remote traffic while it is off +// — and holding them would turn switching federation back on after a month +// into a month of events delivered in one burst. A remote server that +// cares about the room fetches what it missed when it next hears from it. +func (q *Queue) EventPersisted(_ context.Context, n roomserver.Notification) { + if n.Origin != "" && !n.Relay { + return + } + q.mu.Lock() + if !q.running { + q.mu.Unlock() + return + } + q.incoming = append(q.incoming, n) + q.mu.Unlock() + select { + case q.wake <- struct{}{}: + default: + } +} + +// dispatch turns persisted events into queue rows. +func (q *Queue) dispatch() { + defer q.wg.Done() + for { + select { + case <-q.ctx.Done(): + return + case <-q.wake: + } + q.mu.Lock() + batch := q.incoming + q.incoming = nil + q.mu.Unlock() + + for _, n := range batch { + if err := q.queueEvent(q.ctx, n); err != nil && q.ctx.Err() == nil { + q.log.Error("could not queue an event for federation", + "room_id", n.RoomID, "event_id", n.Event.EventID(), "error", err) + } + } + } +} + +// queueEvent writes one event's queue rows. +func (q *Queue) queueEvent(ctx context.Context, n roomserver.Notification) error { + servers, err := q.rooms.JoinedServers(ctx, n.RoomID) + if err != nil { + return err + } + // A membership event also goes to the server of the user it is about: + // a server whose last user was just kicked is no longer joined, and is + // exactly the one that needs to hear it. + if key, ok := n.Event.StateKey(); ok && n.Event.Type() == event.TypeMember { + if server, err := id.ServerNameOf(key); err == nil && !slices.Contains(servers, server) { + servers = append(servers, server) + } + } + + var items []storage.QueuedItem + for _, server := range servers { + if server == q.serverName || server == n.Origin || !q.policy.Allowed(server) { + continue + } + items = append(items, storage.QueuedItem{ + Destination: server, + Kind: storage.QueueKindPDU, + RoomID: n.RoomID, + EventID: n.Event.EventID(), + Payload: n.Event.JSON(), + QueuedAtMS: q.now().UnixMilli(), + }) + } + return q.enqueue(ctx, items) +} + +// QueueEDU queues an EDU for a set of destinations. +func (q *Queue) QueueEDU(ctx context.Context, destinations []string, edu federation.EDU) { + if !q.Running() { + return + } + payload, err := json.Marshal(edu) + if err != nil { + q.log.Error("could not encode an EDU", "type", edu.Type, "error", err) + return + } + var items []storage.QueuedItem + for _, server := range destinations { + if server == q.serverName || !q.policy.Allowed(server) { + continue + } + items = append(items, storage.QueuedItem{ + Destination: server, + Kind: storage.QueueKindEDU, + Payload: payload, + QueuedAtMS: q.now().UnixMilli(), + }) + } + if err := q.enqueue(ctx, items); err != nil { + q.log.Error("could not queue an EDU", "type", edu.Type, "error", err) + } +} + +// enqueue writes rows and wakes their workers. +func (q *Queue) enqueue(ctx context.Context, items []storage.QueuedItem) error { + if len(items) == 0 { + return nil + } + if _, err := q.store.Enqueue(ctx, items); err != nil { + return err + } + seen := map[string]bool{} + for _, item := range items { + if !seen[item.Destination] { + seen[item.Destination] = true + q.kick(item.Destination) + } + } + return nil +} + +// kick wakes a destination's worker, starting one if there is none. +func (q *Queue) kick(destination string) { + q.mu.Lock() + defer q.mu.Unlock() + if !q.running { + return + } + w, ok := q.workers[destination] + if !ok { + w = &worker{destination: destination, wake: make(chan struct{}, 1)} + q.workers[destination] = w + q.wg.Add(1) + go q.run(w) + } + select { + case w.wake <- struct{}{}: + default: + } +} + +// run is one destination's worker. It sends until the queue for the +// destination is empty, then exits; the next enqueue starts a new one. +func (q *Queue) run(w *worker) { + defer q.wg.Done() + for { + dest, err := q.store.Destination(q.ctx, w.destination) + if err != nil { + if q.ctx.Err() != nil { + return + } + q.log.Error("could not read a destination's backoff", "destination", w.destination, "error", err) + dest = storage.Destination{ServerName: w.destination} + } + if wait := time.Duration(dest.RetryAtMS-q.now().UnixMilli()) * time.Millisecond; wait > 0 { + // A wake-up during the wait re-reads the backoff, which is how + // an administrator's reset takes effect at once; an ordinary + // enqueue finds the backoff unchanged and waits again. + timer := time.NewTimer(wait) + select { + case <-q.ctx.Done(): + timer.Stop() + return + case <-w.wake: + timer.Stop() + continue + case <-timer.C: + } + } + + sent, err := q.sendOnce(w.destination, dest) + if q.ctx.Err() != nil { + return + } + if err != nil { + // A failed send has recorded its backoff, which the next pass + // waits out. The pause here is for the other kind of failure — + // the queue's own database — which records nothing and would + // otherwise be retried in a tight loop. + select { + case <-q.ctx.Done(): + return + case <-time.After(time.Second): + } + continue + } + if !sent && q.retire(w) { + return + } + } +} + +// retire removes an idle worker, unless something was queued for it in +// the meantime. +func (q *Queue) retire(w *worker) bool { + q.mu.Lock() + defer q.mu.Unlock() + select { + case <-w.wake: + return false + default: + } + if q.workers[w.destination] == w { + delete(q.workers, w.destination) + } + return true +} + +// sendOnce sends one transaction. It reports whether there was anything to +// send. +func (q *Queue) sendOnce(destination string, health storage.Destination) (bool, error) { + pdus, err := q.store.Queued(q.ctx, destination, storage.QueueKindPDU, maxPDUsPerTxn) + if err != nil { + return false, err + } + edus, err := q.store.Queued(q.ctx, destination, storage.QueueKindEDU, maxEDUsPerTxn) + if err != nil { + return false, err + } + if len(pdus) == 0 && len(edus) == 0 { + return false, nil + } + + // A destination the policy no longer allows gets nothing, and what + // was queued for it is dropped: the allow and deny lists can change at + // runtime, and a backlog for a server that was just denied is exactly + // what must not go out. + if !q.policy.Allowed(destination) { + return true, q.store.Dequeue(q.ctx, append(itemIDs(pdus), itemIDs(edus)...)) + } + + txn := federation.Transaction{ + Origin: q.serverName, + OriginServerTS: q.now().UnixMilli(), + PDUs: make([]json.RawMessage, 0, len(pdus)), + } + for _, item := range pdus { + txn.PDUs = append(txn.PDUs, item.Payload) + } + for _, item := range edus { + var edu federation.EDU + if err := json.Unmarshal(item.Payload, &edu); err == nil { + txn.EDUs = append(txn.EDUs, edu) + } + } + + ctx, cancel := context.WithTimeout(q.ctx, sendTimeout) + defer cancel() + resp, err := q.sender.SendTransaction(ctx, destination, q.nextTxnID(), txn) + if err != nil { + if q.ctx.Err() != nil { + return true, err + } + q.recordFailure(destination, health, err) + return true, err + } + + for eventID, result := range resp.PDUs { + if result.Error != "" { + q.log.Info("a destination refused an event", + "destination", destination, "event_id", eventID, "error", result.Error) + } + } + if err := q.store.Dequeue(q.ctx, append(itemIDs(pdus), itemIDs(edus)...)); err != nil { + return true, err + } + if health.Failures > 0 || health.LastSuccessMS == 0 { + q.log.Info("destination is reachable", "destination", destination, "after_failures", health.Failures) + if err := q.store.SetDestination(q.ctx, storage.Destination{ + ServerName: destination, + LastSuccessMS: q.now().UnixMilli(), + LastFailureMS: health.LastFailureMS, + LastError: health.LastError, + }); err != nil { + q.log.Warn("could not record a destination's recovery", "destination", destination, "error", err) + } + } + return true, nil +} + +// recordFailure backs a destination off. +func (q *Queue) recordFailure(destination string, health storage.Destination, err error) { + failures := health.Failures + 1 + backoff := q.backoff(failures) + now := q.now() + q.log.Info("sending to a destination failed", + "destination", destination, "failures", failures, "retry_in", backoff, "error", err) + if serr := q.store.SetDestination(q.ctx, storage.Destination{ + ServerName: destination, + Failures: failures, + RetryAtMS: now.Add(backoff).UnixMilli(), + LastSuccessMS: health.LastSuccessMS, + LastFailureMS: now.UnixMilli(), + LastError: truncate(err.Error(), 500), + }); serr != nil { + q.log.Warn("could not record a destination's backoff", "destination", destination, "error", serr) + } +} + +// backoff is the wait after a number of consecutive failures: doubling +// from the minimum up to the cap, with a fifth of jitter so that a server +// coming back is not hit by every other server's retry at once. +func (q *Queue) backoff(failures int) time.Duration { + d := q.minBackoff + for i := 1; i < failures && d < q.maxBackoff; i++ { + d *= 2 + } + if d > q.maxBackoff { + d = q.maxBackoff + } + jitter := time.Duration(rand.Int64N(int64(d)/5 + 1)) + return d + jitter +} + +// nextTxnID mints a transaction ID. +func (q *Queue) nextTxnID() string { + q.mu.Lock() + defer q.mu.Unlock() + q.txnSeq++ + return q.txnPrefix + "_" + strconv.FormatUint(q.txnSeq, 10) +} + +// Destinations lists every destination the queue knows, for the admin API. +func (q *Queue) Destinations(ctx context.Context) ([]storage.DestinationSummary, error) { + return q.store.Destinations(ctx) +} + +// ResetBackoff clears a destination's backoff so that it is tried now, +// for an administrator who knows it is back. +func (q *Queue) ResetBackoff(ctx context.Context, destination string) error { + health, err := q.store.Destination(ctx, destination) + if err != nil { + return err + } + health.Failures = 0 + health.RetryAtMS = 0 + if err := q.store.SetDestination(ctx, health); err != nil { + return err + } + q.kick(destination) + return nil +} + +func itemIDs(items []storage.QueuedItem) []int64 { + out := make([]int64, 0, len(items)) + for _, item := range items { + out = append(out, item.ID) + } + return out +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} diff --git a/server/internal/federationapi/queue_test.go b/server/internal/federationapi/queue_test.go new file mode 100644 index 0000000..79312d3 --- /dev/null +++ b/server/internal/federationapi/queue_test.go @@ -0,0 +1,316 @@ +package federationapi + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "path/filepath" + "slices" + "sync" + "testing" + "time" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" + _ "github.com/dariuszjaniak/warp/server/internal/storage/drivers" +) + +const ourName = "warp.example" + +// fakeSender records transactions and fails on demand. +type fakeSender struct { + mu sync.Mutex + txns map[string][]federation.Transaction + ids map[string]bool + down map[string]bool +} + +func newFakeSender() *fakeSender { + return &fakeSender{txns: map[string][]federation.Transaction{}, ids: map[string]bool{}, down: map[string]bool{}} +} + +func (f *fakeSender) SendTransaction(_ context.Context, destination, txnID string, txn federation.Transaction) (federation.SendTransactionResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.down[destination] { + return federation.SendTransactionResponse{}, errors.New("connection refused") + } + if f.ids[txnID] { + return federation.SendTransactionResponse{}, fmt.Errorf("transaction ID %s reused", txnID) + } + f.ids[txnID] = true + f.txns[destination] = append(f.txns[destination], txn) + return federation.SendTransactionResponse{PDUs: map[string]federation.PDUResult{}}, nil +} + +func (f *fakeSender) setDown(destination string, down bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.down[destination] = down +} + +// pdus returns how many PDUs a destination has received. +func (f *fakeSender) pdus(destination string) int { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + for _, txn := range f.txns[destination] { + n += len(txn.PDUs) + } + return n +} + +func (f *fakeSender) largestTxn(destination string) int { + f.mu.Lock() + defer f.mu.Unlock() + largest := 0 + for _, txn := range f.txns[destination] { + largest = max(largest, len(txn.PDUs)) + } + return largest +} + +func (f *fakeSender) edus(destination string) []federation.EDU { + f.mu.Lock() + defer f.mu.Unlock() + var out []federation.EDU + for _, txn := range f.txns[destination] { + out = append(out, txn.EDUs...) + } + return out +} + +// fakeRooms answers JoinedServers from a table. +type fakeRooms map[string][]string + +func (f fakeRooms) JoinedServers(_ context.Context, roomID string) ([]string, error) { + return f[roomID], nil +} + +func openStore(t *testing.T) storage.FederationStore { + t.Helper() + db, err := storage.Open(t.Context(), storage.Config{ + Driver: storage.DriverSQLite, + ConnectionString: "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "q.db")) + "?_pragma=busy_timeout(5000)", + }, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + if err := db.Migrate(t.Context()); err != nil { + t.Fatal(err) + } + return db.Federation() +} + +func newTestQueue(t *testing.T, store storage.FederationStore, sender Sender, rooms RoomServers, policy *federation.Policy) *Queue { + t.Helper() + q := NewQueue(QueueOptions{ + ServerName: ourName, + Store: store, + Sender: sender, + Rooms: rooms, + Policy: policy, + MinBackoff: 20 * time.Millisecond, + MaxBackoff: 100 * time.Millisecond, + Logger: slog.New(slog.DiscardHandler), + }) + t.Cleanup(q.Stop) + return q +} + +var testKey = func() *signing.KeyPair { + k, err := signing.Generate("q") + if err != nil { + panic(err) + } + return k +}() + +// notification builds a persisted-event notification for a message, or a +// membership event about target when target is set. +func notification(t *testing.T, roomID, sender, target string, n int) roomserver.Notification { + t.Helper() + b := event.Builder{ + RoomID: roomID, Type: "m.room.message", Sender: sender, + Content: json.RawMessage(fmt.Sprintf(`{"body":"%d"}`, n)), + PrevEvents: []string{"$prev"}, AuthEvents: []string{"$auth"}, Depth: int64(n + 2), + } + if target != "" { + b.Type = event.TypeMember + b.StateKey = &target + b.Content = json.RawMessage(`{"membership":"leave"}`) + } + e, err := b.Build(time.Unix(1_700_000_000+int64(n), 0), testKey, roomversion.MustGet("11")) + if err != nil { + t.Fatal(err) + } + return roomserver.Notification{RoomID: roomID, Event: e, StreamOrdering: int64(n)} +} + +func waitFor(t *testing.T, what string, check func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if check() { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +func TestQueueSendsToTheRoomsServers(t *testing.T) { + sender := newFakeSender() + rooms := fakeRooms{"!r:warp.example": {ourName, "a.example", "b.example"}} + q := newTestQueue(t, openStore(t), sender, rooms, nil) + if err := q.Start(t.Context()); err != nil { + t.Fatal(err) + } + + q.EventPersisted(t.Context(), notification(t, "!r:warp.example", "@me:"+ourName, "", 1)) + // Received from b and not to be relayed: b sends its own events. + received := notification(t, "!r:warp.example", "@x:b.example", "", 2) + received.Origin = "b.example" + q.EventPersisted(t.Context(), received) + // A join handed over by send_join from c: relayed to everybody but c. + relayed := notification(t, "!r:warp.example", "@y:c.example", "", 3) + relayed.Origin, relayed.Relay = "c.example", true + q.EventPersisted(t.Context(), relayed) + // A kick of a user on d, whose server is no longer joined but must + // hear of it. + q.EventPersisted(t.Context(), notification(t, "!r:warp.example", "@me:"+ourName, "@z:d.example", 4)) + + waitFor(t, "deliveries", func() bool { + return sender.pdus("a.example") == 3 && sender.pdus("b.example") == 3 && sender.pdus("d.example") == 1 + }) + if n := sender.pdus("c.example"); n != 0 { + t.Errorf("c received %d PDUs; it is not in the room and its own join was not to be echoed", n) + } + if n := sender.pdus(ourName); n != 0 { + t.Errorf("the queue sent %d PDUs to this server", n) + } +} + +func TestQueueBacksOffAndRetries(t *testing.T) { + sender := newFakeSender() + sender.setDown("a.example", true) + store := openStore(t) + q := newTestQueue(t, store, sender, fakeRooms{"!r:warp.example": {"a.example"}}, nil) + if err := q.Start(t.Context()); err != nil { + t.Fatal(err) + } + + for i := range 120 { + q.EventPersisted(t.Context(), notification(t, "!r:warp.example", "@me:"+ourName, "", i)) + } + waitFor(t, "a recorded failure", func() bool { + d, err := store.Destination(t.Context(), "a.example") + return err == nil && d.Failures >= 2 + }) + + sender.setDown("a.example", false) + waitFor(t, "every PDU delivered", func() bool { return sender.pdus("a.example") == 120 }) + if largest := sender.largestTxn("a.example"); largest > maxPDUsPerTxn { + t.Errorf("a transaction carried %d PDUs, the limit is %d", largest, maxPDUsPerTxn) + } + waitFor(t, "the failure count to clear", func() bool { + d, err := store.Destination(t.Context(), "a.example") + return err == nil && d.Failures == 0 && d.LastSuccessMS > 0 + }) + left, err := store.Queued(t.Context(), "a.example", storage.QueueKindPDU, 10) + if err != nil || len(left) != 0 { + t.Errorf("queue after delivery = %d items, %v", len(left), err) + } +} + +func TestQueueSurvivesARestart(t *testing.T) { + sender := newFakeSender() + sender.setDown("a.example", true) + store := openStore(t) + rooms := fakeRooms{"!r:warp.example": {"a.example"}} + + first := newTestQueue(t, store, sender, rooms, nil) + if err := first.Start(t.Context()); err != nil { + t.Fatal(err) + } + for i := range 5 { + first.EventPersisted(t.Context(), notification(t, "!r:warp.example", "@me:"+ourName, "", i)) + } + waitFor(t, "the events to be queued", func() bool { + items, _ := store.Queued(t.Context(), "a.example", storage.QueueKindPDU, 10) + return len(items) == 5 + }) + first.Stop() + + // While stopped, events are dropped rather than held. + first.EventPersisted(t.Context(), notification(t, "!r:warp.example", "@me:"+ourName, "", 99)) + + sender.setDown("a.example", false) + if err := store.SetDestination(t.Context(), storage.Destination{ServerName: "a.example"}); err != nil { + t.Fatal(err) + } + second := newTestQueue(t, store, sender, rooms, nil) + if err := second.Start(t.Context()); err != nil { + t.Fatal(err) + } + waitFor(t, "the backlog after a restart", func() bool { return sender.pdus("a.example") == 5 }) + time.Sleep(50 * time.Millisecond) + if n := sender.pdus("a.example"); n != 5 { + t.Errorf("a received %d PDUs; the one persisted while stopped should not have been queued", n) + } +} + +func TestQueueHonoursThePolicy(t *testing.T) { + sender := newFakeSender() + policy := federation.NewPolicy(nil, []string{"denied.example"}) + q := newTestQueue(t, openStore(t), sender, fakeRooms{"!r:warp.example": {"ok.example", "denied.example"}}, policy) + if err := q.Start(t.Context()); err != nil { + t.Fatal(err) + } + q.EventPersisted(t.Context(), notification(t, "!r:warp.example", "@me:"+ourName, "", 1)) + waitFor(t, "delivery to the allowed server", func() bool { return sender.pdus("ok.example") == 1 }) + if n := sender.pdus("denied.example"); n != 0 { + t.Errorf("a denied server received %d PDUs", n) + } +} + +func TestEDUsGoToTheRoomsServers(t *testing.T) { + sender := newFakeSender() + rooms := fakeRooms{"!r:warp.example": {ourName, "a.example"}} + q := newTestQueue(t, openStore(t), sender, rooms, nil) + if err := q.Start(t.Context()); err != nil { + t.Fatal(err) + } + edus := NewEDUs(q, rooms) + edus.Typing(t.Context(), "!r:warp.example", "@me:"+ourName, true, 0) + edus.Receipt(t.Context(), "!r:warp.example", "@me:"+ourName, "m.read", "$e", "", 5) + edus.Receipt(t.Context(), "!r:warp.example", "@me:"+ourName, "m.read.private", "$e", "", 5) + edus.ToDevice(t.Context(), "@me:"+ourName, "m.room_key", map[string]map[string]json.RawMessage{ + "@u:a.example": {"DEV": json.RawMessage(`{"k":1}`)}, + "@v:b.example": {"*": json.RawMessage(`{"k":2}`)}, + }) + + waitFor(t, "EDUs", func() bool { return len(sender.edus("a.example")) == 3 && len(sender.edus("b.example")) == 1 }) + var types []string + for _, e := range sender.edus("a.example") { + types = append(types, e.Type) + } + slices.Sort(types) + if !slices.Equal(types, []string{federation.EDUDirectToDev, federation.EDUReceipt, federation.EDUTyping}) { + t.Errorf("EDUs to a = %v; a private receipt must not leave the server", types) + } + var toDevice federation.DirectToDeviceContent + if err := json.Unmarshal(sender.edus("b.example")[0].Content, &toDevice); err != nil { + t.Fatal(err) + } + if _, ok := toDevice.Messages["@u:a.example"]; ok { + t.Error("b was sent a's recipients") + } +} diff --git a/server/internal/federationapi/server.go b/server/internal/federationapi/server.go new file mode 100644 index 0000000..34302a9 --- /dev/null +++ b/server/internal/federationapi/server.go @@ -0,0 +1,661 @@ +package federationapi + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "strconv" + "strings" + "time" + + "github.com/dariuszjaniak/warp/server/internal/buildinfo" + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/httputil" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" + "github.com/dariuszjaniak/warp/server/internal/syncapi" +) + +// maxBody caps a federation request body. A transaction is up to 50 PDUs +// of up to 64 KiB each, plus EDUs; anything past this is not a +// transaction. +const maxBody = 8 << 20 + +// Error codes the federation API uses beyond the Client-Server set. +const ( + errCodeIncompatibleRoomVersion = "M_INCOMPATIBLE_ROOM_VERSION" + errCodeUnableToAuthoriseJoin = "M_UNABLE_TO_AUTHORISE_JOIN" +) + +// EphemeralSink takes in EDUs from other servers. *syncapi.API satisfies +// it. +type EphemeralSink interface { + SetTyping(ctx context.Context, req syncapi.TypingRequest) error + SetReceipt(ctx context.Context, req syncapi.ReceiptRequest) error + SendToDevice(ctx context.Context, req syncapi.SendToDeviceRequest) error +} + +// Profiles answers profile queries for local users. *userapi.API +// satisfies it. +type Profiles interface { + Account(ctx context.Context, localpart string) (storage.Account, error) +} + +// ServerOptions configure the inbound API. +type ServerOptions struct { + ServerName string + Gate *federation.Gate + Keys *federation.KeyRing + Rooms *roomserver.API + Ephemeral EphemeralSink + Profiles Profiles + Store storage.FederationStore + Logger *slog.Logger + Now func() time.Time +} + +// Server is the inbound Server-Server API. +type Server struct { + serverName string + gate *federation.Gate + keys *federation.KeyRing + rooms *roomserver.API + ephemeral EphemeralSink + profiles Profiles + store storage.FederationStore + log *slog.Logger + now func() time.Time +} + +// NewServer builds the inbound API. +func NewServer(opts ServerOptions) (*Server, error) { + switch { + case opts.ServerName == "": + return nil, errors.New("federationapi: ServerName must not be empty") + case opts.Gate == nil, opts.Keys == nil, opts.Rooms == nil, opts.Store == nil: + return nil, errors.New("federationapi: Gate, Keys, Rooms and Store are required") + } + if opts.Logger == nil { + opts.Logger = slog.Default() + } + if opts.Now == nil { + opts.Now = time.Now + } + return &Server{ + serverName: opts.ServerName, + gate: opts.Gate, + keys: opts.Keys, + rooms: opts.Rooms, + ephemeral: opts.Ephemeral, + profiles: opts.Profiles, + store: opts.Store, + log: opts.Logger, + now: opts.Now, + }, nil +} + +// authedHandler is a federation handler that runs after the request has +// been authenticated, with the origin and the body it was signed over. +type authedHandler func(w http.ResponseWriter, r *http.Request, origin string, body []byte) error + +// RegisterRoutes adds every federation endpoint, each behind the gate: with +// the switch off the whole surface answers M_UNRECOGNIZED, as a server +// with no federation API would. +func (s *Server) RegisterRoutes(r *httputil.Router) { + public := func(pattern string, h httputil.Handler) { + r.Handle(pattern, s.gate.ProtectFunc(h)) + } + authed := func(pattern string, h authedHandler) { + r.Handle(pattern, s.gate.ProtectFunc(s.authenticated(h))) + } + + // Keys and version need no authentication: a server has to be able to + // fetch our key before it can check anything we sign. + public("GET /_matrix/key/v2/server", s.handleServerKey) + public("GET /_matrix/key/v2/server/{keyID}", s.handleServerKey) + public("POST /_matrix/key/v2/query", s.handleKeyQuery) + public("GET /_matrix/key/v2/query/{serverName}", s.handleKeyQueryOne) + public("GET /_matrix/key/v2/query/{serverName}/{keyID}", s.handleKeyQueryOne) + public("GET /_matrix/federation/v1/version", s.handleVersion) + + const v1, v2 = "/_matrix/federation/v1", "/_matrix/federation/v2" + authed("PUT "+v1+"/send/{txnID}", s.handleSend) + authed("GET "+v1+"/make_join/{roomID}/{userID}", s.handleMakeJoin) + authed("PUT "+v1+"/send_join/{roomID}/{eventID}", s.handleSendJoinV1) + authed("PUT "+v2+"/send_join/{roomID}/{eventID}", s.handleSendJoinV2) + authed("GET "+v1+"/make_leave/{roomID}/{userID}", s.handleMakeLeave) + authed("PUT "+v1+"/send_leave/{roomID}/{eventID}", s.handleSendLeaveV1) + authed("PUT "+v2+"/send_leave/{roomID}/{eventID}", s.handleSendLeaveV2) + authed("PUT "+v2+"/invite/{roomID}/{eventID}", s.handleInvite) + authed("GET "+v1+"/event/{eventID}", s.handleEvent) + authed("GET "+v1+"/state/{roomID}", s.handleState) + authed("GET "+v1+"/state_ids/{roomID}", s.handleStateIDs) + authed("GET "+v1+"/event_auth/{roomID}/{eventID}", s.handleEventAuth) + authed("GET "+v1+"/backfill/{roomID}", s.handleBackfill) + authed("POST "+v1+"/get_missing_events/{roomID}", s.handleMissingEvents) + authed("GET "+v1+"/query/directory", s.handleQueryDirectory) + authed("GET "+v1+"/query/profile", s.handleQueryProfile) + authed("GET "+v1+"/publicRooms", s.handlePublicRoomsGet) + authed("POST "+v1+"/publicRooms", s.handlePublicRoomsPost) +} + +// authenticated checks X-Matrix and the allow/deny lists before the +// handler runs. +func (s *Server) authenticated(h authedHandler) httputil.Handler { + return func(w http.ResponseWriter, r *http.Request) error { + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxBody)) + if err != nil { + return httputil.NewError(http.StatusRequestEntityTooLarge, httputil.ErrCodeTooLarge, "Request body too large") + } + origin, err := s.keys.AuthenticateRequest(r.Context(), r, body) + if err != nil { + httputil.LoggerFrom(r.Context()).Info("federation request failed authentication", "error", err) + return httputil.NewError(http.StatusUnauthorized, httputil.ErrCodeUnauthorized, "%s", err.Error()) + } + if !s.gate.Policy().Allowed(origin) { + return httputil.Forbidden("Federation with %s is not allowed by this server", origin) + } + return h(w, r, origin, body) + } +} + +// fedError maps a roomserver error onto the Matrix error a remote server +// is sent. +func fedError(err error) error { + var matrixErr *httputil.MatrixError + switch { + case err == nil: + return nil + case errors.As(err, &matrixErr): + return err + case errors.Is(err, roomserver.ErrIncompatibleRoomVersion): + return httputil.NewError(http.StatusBadRequest, errCodeIncompatibleRoomVersion, "%s", err.Error()) + case errors.Is(err, roomserver.ErrNoSuchRoom), errors.Is(err, roomserver.ErrNoSuchEvent), + errors.Is(err, roomserver.ErrNoSuchAlias): + return httputil.NotFound("%s", err.Error()) + case errors.Is(err, roomserver.ErrForbidden), errors.Is(err, roomserver.ErrRejected), + errors.Is(err, roomserver.ErrNotInRoom): + return httputil.Forbidden("%s", err.Error()) + case errors.Is(err, roomserver.ErrBadEvent), errors.Is(err, roomserver.ErrInvalidContent): + return httputil.BadJSON("%s", err.Error()) + case errors.Is(err, roomserver.ErrFederationDisabled): + return httputil.Unrecognized() + case errors.Is(err, roomserver.ErrRemote): + return httputil.NewError(http.StatusBadGateway, httputil.ErrCodeUnknown, "%s", err.Error()) + } + return err +} + +// decode reads an already-read body. +func decode(body []byte, v any) error { + if len(body) == 0 { + return httputil.NotJSON() + } + if err := json.Unmarshal(body, v); err != nil { + return httputil.BadJSON("Invalid request body: %v", err) + } + return nil +} + +// --- keys and version ------------------------------------------------------- + +func (s *Server) handleServerKey(w http.ResponseWriter, _ *http.Request) error { + doc, err := s.keys.OwnKeyResponse() + if err != nil { + return err + } + httputil.WriteRawJSON(w, http.StatusOK, doc) + return nil +} + +// keyQueryResponse is the notary answer. +type keyQueryResponse struct { + ServerKeys []json.RawMessage `json:"server_keys"` +} + +func (s *Server) handleKeyQuery(w http.ResponseWriter, r *http.Request) error { + var req struct { + ServerKeys map[string]json.RawMessage `json:"server_keys"` + } + if err := httputil.DecodeJSON(w, r, &req); err != nil { + return err + } + resp := keyQueryResponse{ServerKeys: []json.RawMessage{}} + for serverName := range req.ServerKeys { + docs, err := s.keys.NotaryResponse(serverName) + if err != nil { + return err + } + resp.ServerKeys = append(resp.ServerKeys, docs...) + } + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} + +func (s *Server) handleKeyQueryOne(w http.ResponseWriter, r *http.Request) error { + docs, err := s.keys.NotaryResponse(r.PathValue("serverName")) + if err != nil { + return err + } + if docs == nil { + docs = []json.RawMessage{} + } + httputil.WriteJSON(w, http.StatusOK, keyQueryResponse{ServerKeys: docs}) + return nil +} + +func (s *Server) handleVersion(w http.ResponseWriter, _ *http.Request) error { + var resp federation.VersionResponse + resp.Server.Name = "Warp" + resp.Server.Version = buildinfo.Version + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} + +// --- transactions ------------------------------------------------------------- + +func (s *Server) handleSend(w http.ResponseWriter, r *http.Request, origin string, body []byte) error { + txnID := r.PathValue("txnID") + if cached, ok, err := s.store.InboundTransaction(r.Context(), origin, txnID); err != nil { + return err + } else if ok { + httputil.WriteRawJSON(w, http.StatusOK, cached) + return nil + } + + var txn federation.Transaction + if err := decode(body, &txn); err != nil { + return err + } + if txn.Origin != "" && txn.Origin != origin { + return httputil.Forbidden("The transaction's origin is %s, but it was signed by %s", txn.Origin, origin) + } + if len(txn.PDUs) > maxPDUsPerTxn || len(txn.EDUs) > maxEDUsPerTxn { + return httputil.BadJSON("A transaction carries at most %d PDUs and %d EDUs", maxPDUsPerTxn, maxEDUsPerTxn) + } + + resp := federation.SendTransactionResponse{PDUs: map[string]federation.PDUResult{}} + for _, raw := range txn.PDUs { + eventID, err := s.rooms.ReceivePDU(r.Context(), origin, raw) + if eventID == "" { + // Unparseable: there is no ID to report a result under. + s.log.Info("dropping an unreadable PDU", "origin", origin, "error", err) + continue + } + result := federation.PDUResult{} + if err != nil { + result.Error = err.Error() + s.log.Info("refused a PDU", "origin", origin, "event_id", eventID, "error", err) + } + resp.PDUs[eventID] = result + } + for _, edu := range txn.EDUs { + s.receiveEDU(r.Context(), origin, edu) + } + + encoded, err := json.Marshal(resp) + if err != nil { + return err + } + if err := s.store.RecordInboundTransaction(r.Context(), origin, txnID, encoded, s.now().UnixMilli()); err != nil { + s.log.Warn("could not record an inbound transaction", "origin", origin, "txn_id", txnID, "error", err) + } + httputil.WriteRawJSON(w, http.StatusOK, encoded) + return nil +} + +// sameServer reports whether a user belongs to the server that sent the +// EDU about them. Nothing in an EDU is signed, so this is the only check +// there is, and without it any server could make anybody appear to type. +func sameServer(userID, origin string) bool { + server, err := id.ServerNameOf(userID) + return err == nil && server == origin +} + +// receiveEDU takes one EDU in. Failures are logged and otherwise ignored: +// an EDU is ephemeral, and the transaction it came in succeeds whatever +// becomes of it. +func (s *Server) receiveEDU(ctx context.Context, origin string, edu federation.EDU) { + if s.ephemeral == nil { + return + } + switch edu.Type { + case federation.EDUTyping: + var c federation.TypingContent + if json.Unmarshal(edu.Content, &c) != nil || !sameServer(c.UserID, origin) || !s.allowed(ctx, c.RoomID, origin) { + return + } + if err := s.ephemeral.SetTyping(ctx, syncapi.TypingRequest{RoomID: c.RoomID, User: c.UserID, Typing: c.Typing}); err != nil { + s.log.Debug("ignored a typing EDU", "origin", origin, "room_id", c.RoomID, "error", err) + } + + case federation.EDUReceipt: + var c federation.ReceiptContent + if json.Unmarshal(edu.Content, &c) != nil { + return + } + for roomID, byType := range c { + if !s.allowed(ctx, roomID, origin) { + continue + } + for receiptType, byUser := range byType { + if receiptType != syncapi.ReceiptRead { + continue + } + for userID, data := range byUser { + if !sameServer(userID, origin) || len(data.EventIDs) == 0 { + continue + } + if err := s.ephemeral.SetReceipt(ctx, syncapi.ReceiptRequest{ + RoomID: roomID, User: userID, ReceiptType: receiptType, + EventID: data.EventIDs[len(data.EventIDs)-1], ThreadID: data.Data.ThreadID, + }); err != nil { + s.log.Debug("ignored a receipt EDU", "origin", origin, "room_id", roomID, "error", err) + } + } + } + } + + case federation.EDUDirectToDev: + var c federation.DirectToDeviceContent + if json.Unmarshal(edu.Content, &c) != nil || !sameServer(c.Sender, origin) { + return + } + if err := s.ephemeral.SendToDevice(ctx, syncapi.SendToDeviceRequest{ + Sender: c.Sender, EventType: c.Type, Messages: c.Messages, + }); err != nil { + s.log.Debug("ignored a to-device EDU", "origin", origin, "error", err) + } + + default: + // Presence and device list updates belong to phases that have not + // landed (presence is off by default; device lists are Phase 7). + s.log.Debug("ignored an EDU", "origin", origin, "type", edu.Type) + } +} + +// --- membership --------------------------------------------------------------- + +func (s *Server) handleMakeJoin(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error { + version, tmpl, err := s.rooms.MakeJoin(r.Context(), origin, r.PathValue("roomID"), r.PathValue("userID"), + r.URL.Query()["ver"]) + if err != nil { + if errors.Is(err, roomserver.ErrForbidden) && strings.Contains(err.Error(), "authorise a restricted join") { + return httputil.NewError(http.StatusBadRequest, errCodeUnableToAuthoriseJoin, "%s", err.Error()) + } + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, federation.MakeJoinResponse{RoomVersion: version, Event: tmpl}) + return nil +} + +func (s *Server) sendJoin(r *http.Request, origin string, body []byte) (federation.SendJoinResponse, error) { + res, err := s.rooms.SendJoin(r.Context(), origin, r.PathValue("roomID"), r.PathValue("eventID"), body) + if err != nil { + return federation.SendJoinResponse{}, fedError(err) + } + return federation.SendJoinResponse{ + Origin: s.serverName, + State: res.State, + AuthChain: res.AuthChain, + Event: res.Event, + ServersInRoom: res.ServersInRoom, + }, nil +} + +func (s *Server) handleSendJoinV2(w http.ResponseWriter, r *http.Request, origin string, body []byte) error { + resp, err := s.sendJoin(r, origin, body) + if err != nil { + return err + } + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} + +// handleSendJoinV1 is the deprecated form, which wraps the same answer in +// a [200, body] pair. +func (s *Server) handleSendJoinV1(w http.ResponseWriter, r *http.Request, origin string, body []byte) error { + resp, err := s.sendJoin(r, origin, body) + if err != nil { + return err + } + httputil.WriteJSON(w, http.StatusOK, []any{http.StatusOK, resp}) + return nil +} + +func (s *Server) handleMakeLeave(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error { + version, tmpl, err := s.rooms.MakeLeave(r.Context(), origin, r.PathValue("roomID"), r.PathValue("userID")) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, federation.MakeJoinResponse{RoomVersion: version, Event: tmpl}) + return nil +} + +func (s *Server) handleSendLeaveV2(w http.ResponseWriter, r *http.Request, origin string, body []byte) error { + if err := s.rooms.SendLeave(r.Context(), origin, r.PathValue("roomID"), r.PathValue("eventID"), body); err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, struct{}{}) + return nil +} + +func (s *Server) handleSendLeaveV1(w http.ResponseWriter, r *http.Request, origin string, body []byte) error { + if err := s.rooms.SendLeave(r.Context(), origin, r.PathValue("roomID"), r.PathValue("eventID"), body); err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, []any{http.StatusOK, struct{}{}}) + return nil +} + +func (s *Server) handleInvite(w http.ResponseWriter, r *http.Request, origin string, body []byte) error { + var req federation.InviteRequest + if err := decode(body, &req); err != nil { + return err + } + signed, err := s.rooms.ReceiveInvite(r.Context(), origin, req.RoomVersion, req.Event, req.InviteRoomState) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, federation.InviteResponse{Event: signed}) + return nil +} + +// --- reads -------------------------------------------------------------------- + +func (s *Server) transaction(pdus []json.RawMessage) federation.Transaction { + if pdus == nil { + pdus = []json.RawMessage{} + } + return federation.Transaction{Origin: s.serverName, OriginServerTS: s.now().UnixMilli(), PDUs: pdus} +} + +func (s *Server) handleEvent(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error { + raw, err := s.rooms.EventForServer(r.Context(), origin, r.PathValue("eventID")) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, s.transaction([]json.RawMessage{raw})) + return nil +} + +func (s *Server) handleState(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error { + eventID := r.URL.Query().Get("event_id") + if eventID == "" { + return httputil.MissingParam("event_id is required") + } + state, chain, err := s.rooms.StateForServer(r.Context(), origin, r.PathValue("roomID"), eventID) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, federation.StateResponse{PDUs: state, AuthChain: chain}) + return nil +} + +func (s *Server) handleStateIDs(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error { + eventID := r.URL.Query().Get("event_id") + if eventID == "" { + return httputil.MissingParam("event_id is required") + } + state, chain, err := s.rooms.StateIDsForServer(r.Context(), origin, r.PathValue("roomID"), eventID) + if err != nil { + return fedError(err) + } + if state == nil { + state = []string{} + } + if chain == nil { + chain = []string{} + } + httputil.WriteJSON(w, http.StatusOK, federation.StateIDsResponse{PDUIDs: state, AuthChainIDs: chain}) + return nil +} + +func (s *Server) handleEventAuth(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error { + chain, err := s.rooms.EventAuthForServer(r.Context(), origin, r.PathValue("roomID"), r.PathValue("eventID")) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, federation.EventAuthResponse{AuthChain: chain}) + return nil +} + +func (s *Server) handleBackfill(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error { + query := r.URL.Query() + from := query["v"] + if len(from) == 0 { + return httputil.MissingParam("v is required") + } + limit, err := strconv.Atoi(query.Get("limit")) + if err != nil || limit <= 0 { + return httputil.InvalidParam("limit must be a positive integer") + } + events, err := s.rooms.BackfillForServer(r.Context(), origin, r.PathValue("roomID"), from, limit) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, s.transaction(events)) + return nil +} + +func (s *Server) handleMissingEvents(w http.ResponseWriter, r *http.Request, origin string, body []byte) error { + var req federation.MissingEventsRequest + if err := decode(body, &req); err != nil { + return err + } + events, err := s.rooms.MissingEventsForServer(r.Context(), origin, r.PathValue("roomID"), + req.EarliestEvents, req.LatestEvents, req.Limit, req.MinDepth) + if err != nil { + return fedError(err) + } + if events == nil { + events = []json.RawMessage{} + } + httputil.WriteJSON(w, http.StatusOK, federation.MissingEventsResponse{Events: events}) + return nil +} + +// --- queries ------------------------------------------------------------------ + +func (s *Server) handleQueryDirectory(w http.ResponseWriter, r *http.Request, _ string, _ []byte) error { + alias := r.URL.Query().Get("room_alias") + if alias == "" { + return httputil.MissingParam("room_alias is required") + } + resolved, err := s.rooms.QueryAliasForServer(r.Context(), alias) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, federation.DirectoryResponse{RoomID: resolved.RoomID, Servers: resolved.Servers}) + return nil +} + +func (s *Server) handleQueryProfile(w http.ResponseWriter, r *http.Request, _ string, _ []byte) error { + query := r.URL.Query() + userID := query.Get("user_id") + localpart, server, err := id.SplitUserID(userID) + if err != nil { + return httputil.InvalidParam("user_id is not a user ID") + } + if server != s.serverName || s.profiles == nil { + return httputil.NotFound("%s is not a user of this server", userID) + } + account, err := s.profiles.Account(r.Context(), localpart) + if errors.Is(err, storage.ErrNoSuchUser) { + return httputil.NotFound("Profile not found") + } else if err != nil { + return err + } + resp := map[string]string{} + switch field := query.Get("field"); field { + case "": + if account.DisplayName != "" { + resp["displayname"] = account.DisplayName + } + if account.AvatarURL != "" { + resp["avatar_url"] = account.AvatarURL + } + case "displayname": + if account.DisplayName != "" { + resp["displayname"] = account.DisplayName + } + case "avatar_url": + if account.AvatarURL != "" { + resp["avatar_url"] = account.AvatarURL + } + default: + return httputil.InvalidParam("field must be displayname or avatar_url") + } + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} + +func (s *Server) handlePublicRoomsGet(w http.ResponseWriter, r *http.Request, _ string, _ []byte) error { + query := r.URL.Query() + limit, _ := strconv.Atoi(query.Get("limit")) + return s.publicRooms(w, r, roomserver.PublicRoomsRequest{Limit: limit, Since: query.Get("since")}) +} + +func (s *Server) handlePublicRoomsPost(w http.ResponseWriter, r *http.Request, _ string, body []byte) error { + var req struct { + Limit int `json:"limit"` + Since string `json:"since"` + Filter struct { + GenericSearchTerm string `json:"generic_search_term"` + } `json:"filter"` + } + if len(body) > 0 { + if err := decode(body, &req); err != nil { + return err + } + } + return s.publicRooms(w, r, roomserver.PublicRoomsRequest{ + Limit: req.Limit, Since: req.Since, Search: req.Filter.GenericSearchTerm, + }) +} + +func (s *Server) publicRooms(w http.ResponseWriter, r *http.Request, req roomserver.PublicRoomsRequest) error { + resp, err := s.rooms.PublicRooms(r.Context(), req) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} + +// String is for logs. +func (s *Server) String() string { return fmt.Sprintf("federation API for %s", s.serverName) } + +// allowed applies a room's server ACL to an EDU about it. An EDU is not +// a room event, but it is still a server speaking in the room, and a +// server the room has banned does not get to make its users appear to +// type. +func (s *Server) allowed(ctx context.Context, roomID, origin string) bool { + ok, err := s.rooms.ServerAllowed(ctx, roomID, origin) + return err == nil && ok +} diff --git a/server/internal/matrix/event/event.go b/server/internal/matrix/event/event.go index 46eef71..3b38a33 100644 --- a/server/internal/matrix/event/event.go +++ b/server/internal/matrix/event/event.go @@ -19,8 +19,8 @@ // can. Warp does not: such an event is rejected whatever its room version. // It only arises for events that were already outside the current grammar // when they were created, and handling them means a second, lenient -// encoder — which can wait until federation (PLAN.md phase 5) makes old -// rooms reachable at all. +// encoder — which waits for a room that needs it: joining the rooms of +// today over federation (PLAN.md phase 5) does not. package event import ( diff --git a/server/internal/matrix/roomversion/roomversion.go b/server/internal/matrix/roomversion/roomversion.go index 8c9ae2e..b21204b 100644 --- a/server/internal/matrix/roomversion/roomversion.go +++ b/server/internal/matrix/roomversion/roomversion.go @@ -19,8 +19,8 @@ // prev_events and auth_events as [id, hashes] tuples rather than plain // lists, and use state resolution v1, which is enough of a separate event // format to be its own piece of work. It is only needed to join rooms -// created before 2018 on other servers, so it waits for federation -// (PLAN.md phase 5) and the registry rejects those rooms clearly until then. +// created before 2018 on other servers, which federation (PLAN.md phase 5) +// does not yet do; the registry rejects those rooms clearly until it does. package roomversion import ( diff --git a/server/internal/matrix/signing/key.go b/server/internal/matrix/signing/key.go index cd485af..e6f5416 100644 --- a/server/internal/matrix/signing/key.go +++ b/server/internal/matrix/signing/key.go @@ -130,7 +130,8 @@ func (k *KeyPair) Save(path string) error { // Load reads a key file written by Save. Lines starting with '#' and blank // lines are ignored; the first key line wins. Older keys (a file with -// several entries) will matter once key rotation lands in Phase 5. +// several entries) will matter once key rotation exists, which it does not +// yet: see the known gaps in docs/federation.md. func Load(path string) (*KeyPair, error) { data, err := os.ReadFile(path) if err != nil { diff --git a/server/internal/roomserver/acl.go b/server/internal/roomserver/acl.go new file mode 100644 index 0000000..473fc93 --- /dev/null +++ b/server/internal/roomserver/acl.go @@ -0,0 +1,138 @@ +package roomserver + +import ( + "context" + "encoding/json" + "fmt" + "net" + "regexp" + "strings" + "sync" + + "github.com/dariuszjaniak/warp/server/internal/federation" +) + +// TypeServerACL is the state event that says which servers may take part +// in a room. +const TypeServerACL = "m.room.server_acl" + +// Server ACLs. +// +// A room can ban whole servers. The ACL is ordinary room state, and what +// it does is decided by every server in the room for itself: requests and +// events from a server the ACL denies are refused, as if that server were +// not there. It is the one defence a room has against a server that +// creates users faster than moderators can ban them. +// +// It applies to what a server *sends*: its transactions' PDUs, its EDUs, +// its joins and leaves, and its reads of the room's history. It does not +// rewrite the past — events already in the room stay — and it does not +// apply to the server enforcing it. + +// aclContent is the content of an m.room.server_acl event. +type aclContent struct { + Allow []string `json:"allow"` + Deny []string `json:"deny"` + AllowIPLiterals *bool `json:"allow_ip_literals"` +} + +// globCache keeps compiled ACL globs: the same handful of patterns are +// matched against every request a busy room sees. +var globCache sync.Map // pattern -> *regexp.Regexp + +// globMatch matches a server name against an ACL glob, where '*' is any +// run of characters and '?' is any one. The match ignores case, as every +// implementation does: a DNS name is not case-sensitive, and an ACL that +// "example.com" got around by writing "EXAMPLE.com" would be no ACL. +func globMatch(pattern, name string) bool { + if cached, ok := globCache.Load(pattern); ok { + return cached.(*regexp.Regexp).MatchString(name) + } + var b strings.Builder + b.WriteString(`(?i)^`) + for _, r := range pattern { + switch r { + case '*': + b.WriteString(`.*`) + case '?': + b.WriteString(`.`) + default: + b.WriteString(regexp.QuoteMeta(string(r))) + } + } + b.WriteString(`$`) + re, err := regexp.Compile(b.String()) + if err != nil { + return false + } + globCache.Store(pattern, re) + return re.MatchString(name) +} + +// aclAllows applies one ACL to a server name. The port is not part of +// what an ACL matches. +func aclAllows(acl aclContent, serverName string) bool { + host := serverName + if parsed, err := federation.ParseServerName(serverName); err == nil { + host = parsed.Host + if parsed.IsIP && acl.AllowIPLiterals != nil && !*acl.AllowIPLiterals { + return false + } + } else if h, _, err := net.SplitHostPort(serverName); err == nil { + host = h + } + for _, deny := range acl.Deny { + if globMatch(deny, host) { + return false + } + } + for _, allow := range acl.Allow { + if globMatch(allow, host) { + return true + } + } + // An ACL with an empty allow list allows nobody. That is what the + // spec says, and the room's own server is protected from it only + // because a server never applies a room's ACL to itself. + return false +} + +// ServerAllowed reports whether a room's ACL lets a server take part. A +// room with no ACL allows everybody, and this server is always allowed. +func (r *API) ServerAllowed(ctx context.Context, roomID, serverName string) (bool, error) { + if serverName == r.serverName { + return true, nil + } + stored, err := r.store.CurrentStateEvent(ctx, roomID, TypeServerACL, "") + if isNotFound(err) { + return true, nil + } else if err != nil { + return false, err + } + acl, ok := parseACL(contentOf(stored)) + if !ok { + // An ACL nobody can read restricts nobody; refusing everyone over + // a malformed event would let one bad state event cut a room off. + r.log.Warn("ignoring an unreadable server ACL", "room_id", roomID, "event_id", stored.EventID) + return true, nil + } + return aclAllows(acl, serverName), nil +} + +// parseACL decodes an ACL's content, reporting whether it could. +func parseACL(content json.RawMessage) (aclContent, bool) { + var acl aclContent + return acl, json.Unmarshal(content, &acl) == nil +} + +// requireAllowed refuses a server the room's ACL denies. +func (r *API) requireAllowed(ctx context.Context, roomID, serverName string) error { + ok, err := r.ServerAllowed(ctx, roomID, serverName) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("%w: %s is denied by the server ACL of %s", ErrForbidden, serverName, roomID) + } + return nil +} diff --git a/server/internal/roomserver/acl_internal_test.go b/server/internal/roomserver/acl_internal_test.go new file mode 100644 index 0000000..429d34f --- /dev/null +++ b/server/internal/roomserver/acl_internal_test.go @@ -0,0 +1,41 @@ +package roomserver + +import "testing" + +func TestACLMatching(t *testing.T) { + no := false + acl := aclContent{ + Allow: []string{"*"}, + Deny: []string{"evil.example", "*.bad.example", "spam?.example"}, + AllowIPLiterals: &no, + } + cases := map[string]bool{ + "good.example": true, + "GOOD.example": true, + "evil.example": false, + "EVIL.EXAMPLE": false, + "evil.example:8448": false, // the port is not part of the match + "a.bad.example": false, + "bad.example": true, // "*.bad.example" needs a subdomain + "spam1.example": false, + "spam12.example": true, + "1.2.3.4": false, + "[::1]:8448": false, + "notevil.example": true, + "evil.example.attacker": true, + } + for server, want := range cases { + if got := aclAllows(acl, server); got != want { + t.Errorf("aclAllows(%q) = %v, want %v", server, got, want) + } + } + + // An empty allow list allows nobody. + if aclAllows(aclContent{}, "anyone.example") { + t.Error("an ACL with no allow list let a server in") + } + // IP literals are allowed unless the ACL says otherwise. + if !aclAllows(aclContent{Allow: []string{"*"}}, "1.2.3.4") { + t.Error("an IP literal was refused by an ACL that does not mention them") + } +} diff --git a/server/internal/roomserver/backfill.go b/server/internal/roomserver/backfill.go new file mode 100644 index 0000000..b751562 --- /dev/null +++ b/server/internal/roomserver/backfill.go @@ -0,0 +1,323 @@ +package roomserver + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "sync" + + "github.com/dariuszjaniak/warp/server/internal/matrix/auth" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/matrix/stateres" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Outbound backfill: asking other servers for a room's history. +// +// A room joined over federation starts, on this server, at the join. The +// join is handed the room's state, which is enough to take part, but none +// of the events that led to it — so a client scrolling up reaches the join +// and stops, in a room that may have years of history on every other +// server in it. Backfill is what fills that in: when /messages pages back +// past the oldest event held, the servers in the room are asked for what +// came before it (GET /backfill), and what they send is checked like any +// other event from another server and stored *below* everything already +// here. +// +// "Below" is the whole difficulty. The stream ordering is the order clients +// are shown a room in, and its counter only goes up, because every event it +// has ever numbered arrived after the ones before it. History arrives last +// and belongs first, so it is numbered by a second counter that runs +// downwards from -1 (storage.RoomStore.WriteBackfill). Nothing streams +// through those positions: a live /sync is "everything after my token", +// which never reaches below zero, so backfilled history reaches clients +// only through /messages, /context and the state at a point in it — which +// is right, because it is not news. +// +// A backfilled event goes through the same checks as one that arrived +// live, but one: it is not checked against the room's current state. Soft +// failure is about refusing events built on a state the room has since +// moved past, and history is by definition built on a state the room has +// since moved past. + +// Bounds on what one page of /messages can make this server fetch. +const ( + // backfillMinBatch is the fewest events one /backfill request asks + // for, however small the page. A request costs a round trip and a + // state fetch at its far end whatever its size, so asking for a + // handful at a time would make scrolling a room's history slow for no + // saving. + backfillMinBatch = 20 + + // backfillRounds is how many /backfill requests one page may make + // before it is returned short. Events a server sends that fail the + // checks, or that the page's filter leaves out, are why one round may + // not be enough. + backfillRounds = 3 + + // backfillFrontierScan is how many of the room's oldest events are + // read to find where its history stops. + backfillFrontierScan = 50 +) + +// batchEvent is an event of a backfill batch that has passed the checks, +// with the state after it: what the events after it in the batch are +// checked against. +type batchEvent struct { + event *event.Event + stateAfter []*event.Event +} + +// wantsBackfill reports whether a page of a room's history came back short +// at the bottom of what this server holds, where other servers may hold +// more. +func (r *API) wantsBackfill(req TimelineRequest, visible VisibleRange, page Timeline) bool { + if !req.Backfill || req.Direction != storage.DirectionBackward || page.Limited || !r.federating() { + return false + } + // Only a window that is open at the bottom — no `to` from the client, + // and a history visibility that lets the user see before their join — + // can reach history this server does not have. + if low, _, ok := visible.Clamp(req.Low, req.High); !ok || low != 0 { + return false + } + // A page that reaches the create event has reached the beginning of + // the room. + return len(page.Events) == 0 || page.Events[0].Type != event.TypeCreate +} + +// backfillLock serialises the backfills of one room, so that two clients +// scrolling the same room at once do not fetch the same history twice. +// It is not the room's write lock: a backfill spends most of its time +// waiting for other servers, and live events must not wait with it. +func (r *API) backfillLock(roomID string) *sync.Mutex { + actual, _ := r.backfills.LoadOrStore(roomID, &sync.Mutex{}) + mu, ok := actual.(*sync.Mutex) + if !ok { + // Unreachable: nothing else ever stores into this map. + panic("roomserver: backfill lock map holds something that is not a mutex") + } + return mu +} + +// backfill asks the servers in a room for the history before the oldest +// event this server holds, and stores what passes below it. It returns how +// many events were added; zero with no error means there was nothing to +// ask for, or nobody had anything. +func (r *API) backfill(ctx context.Context, room storage.Room, rules roomversion.Rules, want int) (int, error) { + mu := r.backfillLock(room.ID) + mu.Lock() + defer mu.Unlock() + + // Worked out under the lock: a backfill that finished while this one + // waited has moved it. + frontier, err := r.backfillFrontier(ctx, room.ID, rules) + if err != nil || len(frontier) == 0 { + return 0, err + } + limit := min(max(want, backfillMinBatch), maxBackfillLimit) + + var lastErr error + for _, server := range r.candidateServers(ctx, room.ID, nil) { + if err := r.requireFederation("backfilling a room"); err != nil { + return 0, err + } + txn, err := r.remote.Backfill(ctx, server, room.ID, frontier, limit) + if err != nil { + lastErr = remoteError(server, err) + r.log.Info("a server could not backfill a room", "room_id", room.ID, "server", server, "error", err) + continue + } + added, err := r.acceptBackfill(ctx, server, room, rules, txn.PDUs) + if err != nil { + return 0, err + } + if added > 0 { + return added, nil + } + // A server that joined after this one did may have none of the + // history; the next one may. + } + return 0, lastErr +} + +// backfillFrontier finds where a room's history stops on this server: the +// prev_events of its oldest timeline events that are not themselves in the +// timeline. They are what /backfill is asked to start from. +// +// The oldest events, not only the oldest one: a room's history is a graph, +// and a batch that ends partway down two branches leaves an edge on each. +// Reading a bounded number of them keeps this one query however long the +// room is; the edges it misses are the ones a gap left higher up, which +// are not history. +func (r *API) backfillFrontier(ctx context.Context, roomID string, rules roomversion.Rules) ([]string, error) { + oldest, err := r.store.Timeline(ctx, storage.TimelineRequest{ + RoomID: roomID, + Direction: storage.DirectionForward, + Limit: backfillFrontierScan, + }) + if err != nil { + return nil, err + } + inScan := make(map[string]bool, len(oldest)) + for _, s := range oldest { + inScan[s.EventID] = true + } + var prevs []string + for _, s := range oldest { + e, err := parseEvent(s, rules) + if err != nil { + return nil, err + } + for _, id := range e.PrevEventIDs() { + if !inScan[id] && !slices.Contains(prevs, id) { + prevs = append(prevs, id) + } + } + } + if len(prevs) == 0 { + return nil, nil + } + + // An outlier is held without its place in the timeline, so it is still + // history to ask for; an event in the timeline further up is not. + held, err := r.store.EventsByID(ctx, roomID, prevs) + if err != nil { + return nil, err + } + for _, s := range held { + if !s.Outlier { + prevs = slices.DeleteFunc(prevs, func(id string) bool { return id == s.EventID }) + } + } + return prevs, nil +} + +// acceptBackfill checks the events a /backfill answered with, oldest +// first, and stores the ones that pass in one write. +// +// Each goes through the intake's steps 1 to 3 (see federated.go): it +// verifies, it passes the rules against its own auth events, and it passes +// them against the state before it. That state comes from the events +// before it in the same batch where it can, and otherwise from the state +// at its prev_events as the sending server reports it — which, for the +// oldest events of a batch, is a /state_ids request: the price of having a +// state snapshot for every backfilled event, which is what lets a client +// be told the room's state at any point of its history. +// +// An event that fails is left out and logged. It is not an error: the rest +// of the batch may still be good, and a server that sends bad history is +// one this server will not take history from, not one that breaks +// /messages. +func (r *API) acceptBackfill(ctx context.Context, origin string, room storage.Room, rules roomversion.Rules, raws []json.RawMessage) (int, error) { + var events []*event.Event + seen := map[string]bool{} + for _, raw := range raws { + e, err := parseRemote(raw, rules) + if err != nil { + r.log.Debug("ignoring an unparseable backfilled event", "room_id", room.ID, "origin", origin, "error", err) + continue + } + if !inRoom(e, room.ID) || seen[e.EventID()] { + continue + } + seen[e.EventID()] = true + events = append(events, e) + } + + // What is already in the timeline needs nothing; an outlier is checked + // again, because it is about to be given a place in the timeline and + // a state snapshot, which it never had. + ids := make([]string, 0, len(events)) + for _, e := range events { + ids = append(ids, e.EventID()) + } + held, err := r.store.EventsByID(ctx, room.ID, ids) + if err != nil { + return 0, err + } + for _, s := range held { + if !s.Outlier { + events = slices.DeleteFunc(events, func(e *event.Event) bool { return e.EventID() == s.EventID }) + } + } + if len(events) == 0 { + return 0, nil + } + + eventsByDepth(events) + verified := r.verifyAll(ctx, events) + create, err := r.createEvent(ctx, room, rules) + if err != nil { + return 0, err + } + + batch := map[string]batchEvent{} + writes := make([]storage.BackfillEvent, 0, len(verified)) + rejected := 0 + for _, e := range verified { + in := incoming{origin: origin, room: room, rules: rules, event: e, batch: batch} + stateBefore, err := r.checkBackfilled(ctx, in, create) + if err != nil { + rejected++ + r.log.Debug("a backfilled event was not accepted", + "room_id", room.ID, "origin", origin, "event_id", e.EventID(), "type", e.Type(), "error", err) + continue + } + after := stateres.Sorted(stateBefore) + if e.IsState() { + after = append(after, e) + } + batch[e.EventID()] = batchEvent{event: e, stateAfter: after} + writes = append(writes, storage.BackfillEvent{ + Event: storedOf(room.ID, e), + StateBefore: stateEntries(stateBefore), + }) + } + if rejected > 0 || len(verified) < len(events) { + r.log.Info("refused events in a backfill", + "room_id", room.ID, "origin", origin, + "unverified", len(events)-len(verified), "rejected", rejected) + } + + if err := r.store.WriteBackfill(ctx, room.ID, writes); err != nil { + return 0, err + } + return len(writes), nil +} + +// checkBackfilled runs one backfilled event through the intake's checks and +// returns the state before it. +func (r *API) checkBackfilled(ctx context.Context, in incoming, create *event.Event) (auth.State, error) { + e := in.event + + authEvents, err := r.authEventsOf(ctx, in) + if err != nil { + return nil, err + } + if err := auth.CheckAuthEvents(e, authEvents); err != nil { + return nil, err + } + if err := auth.Check(ctx, e, withCreate(auth.NewState(authEvents), create), r.verifier()); err != nil { + return nil, err + } + + // The create event is the one event with nothing before it. + if e.Type() == event.TypeCreate && len(e.PrevEventIDs()) == 0 { + if create == nil || e.EventID() != create.EventID() { + return nil, fmt.Errorf("%w: a second create event", ErrBadEvent) + } + return auth.State{}, nil + } + + stateBefore, err := r.stateBeforeIncoming(ctx, in) + if err != nil { + return nil, err + } + if err := auth.Check(ctx, e, withCreate(stateBefore, create), r.verifier()); err != nil { + return nil, fmt.Errorf("%w (against the state before it)", err) + } + return stateBefore, nil +} diff --git a/server/internal/roomserver/backfill_test.go b/server/internal/roomserver/backfill_test.go new file mode 100644 index 0000000..a7bc425 --- /dev/null +++ b/server/internal/roomserver/backfill_test.go @@ -0,0 +1,222 @@ +package roomserver_test + +import ( + "context" + "fmt" + "slices" + "testing" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Outbound backfill: a server that joined a room part-way through filling +// in the history before its join when a client scrolls back to it. + +func (t transport) Backfill(ctx context.Context, dest, roomID string, from []string, limit int) (federation.Transaction, error) { + api, err := t.to(dest) + if err != nil { + return federation.Transaction{}, err + } + pdus, err := api.BackfillForServer(ctx, t.from.name, roomID, from, limit) + if err != nil { + return federation.Transaction{}, httpError(err) + } + return federation.Transaction{Origin: dest, PDUs: pdus}, nil +} + +// scrollBack pages a room's history backwards the way a client scrolling +// up does — /messages from the end, then from each page's `end` token — +// and returns every event ID it was shown, oldest first. +func (p *peer) scrollBack(t *testing.T, roomID, user string, pageSize int) []string { + t.Helper() + var ids []string + var high int64 + for range 100 { + page, err := p.api.Timeline(t.Context(), roomserver.TimelineRequest{ + RoomID: roomID, User: user, High: high, + Direction: storage.DirectionBackward, Limit: pageSize, Backfill: true, + }) + if err != nil { + t.Fatalf("%s: Timeline: %v", p.name, err) + } + page.Events = slices.Clone(page.Events) + slices.Reverse(page.Events) + for _, e := range page.Events { + ids = append(ids, e.EventID) + } + if !page.Limited { + slices.Reverse(ids) + return ids + } + high = page.Start + } + t.Fatal("scrolling back never reached the end of the room") + return nil +} + +func TestBackfillFillsInTheHistoryBeforeAJoin(t *testing.T) { + for _, version := range []string{"10", "11", "12"} { + t.Run("v"+version, func(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{ + Creator: userA, Preset: roomserver.PresetPublicChat, Name: "Federated", Version: version, + }) + if err != nil { + t.Fatalf("CreateRoom: %v", err) + } + roomID := resp.RoomID + + // Enough history that it takes several /backfill requests, + // and a state change in the middle of it. + for i := range 25 { + a.send(t, roomID, userA, fmt.Sprint("before ", i)) + } + topicID, err := a.api.SendEvent(t.Context(), roomserver.SendEventRequest{ + RoomID: roomID, Sender: userA, Type: roomserver.TypeTopic, StateKey: ptrTo(""), + Content: []byte(`{"topic":"history"}`), + }) + if err != nil { + t.Fatalf("SendEvent: %v", err) + } + for i := range 10 { + a.send(t, roomID, userA, fmt.Sprint("after the topic ", i)) + } + a.takeOutbox() // nobody else is in the room yet + + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{ + RoomID: roomID, User: userB, ServerNames: []string{serverA}, + }); err != nil { + t.Fatalf("remote join: %v", err) + } + a.send(t, roomID, userA, "after the join") + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + + // Before anybody scrolls, B's timeline starts at its join. + if held := b.timelineIDs(t, roomID); len(held) != 2 { + t.Fatalf("B's timeline before backfill = %d events, want the join and one message", len(held)) + } + top, err := b.api.StreamTop(t.Context()) + if err != nil { + t.Fatalf("StreamTop: %v", err) + } + + // Scrolling back on B shows the whole room, in A's order. + want := a.timelineIDs(t, roomID) + got := b.scrollBack(t, roomID, userB, 10) + if !slices.Equal(got, want) { + t.Fatalf("B's scrollback has %d events, A's room %d:\nB: %v\nA: %v", len(got), len(want), got, want) + } + + // The history is in B's timeline proper, below everything B + // held, and is not news: the top of the stream — what a live + // /sync reads from — has not moved. + create, err := b.store.Event(t.Context(), roomID, want[0]) + if err != nil { + t.Fatalf("Event(create): %v", err) + } + if create.Outlier || create.StreamOrdering >= 0 || create.Type != event.TypeCreate { + t.Errorf("B's create event: type %s, position %d, outlier %v; want a timeline event below zero", + create.Type, create.StreamOrdering, create.Outlier) + } + if after, err := b.api.StreamTop(t.Context()); err != nil { + t.Fatalf("StreamTop: %v", err) + } else if after != top { + t.Errorf("backfill moved the stream top from %d to %d", top, after) + } + + // The room's state at a point inside the history is the state + // then: no topic before the topic was set, the topic after. + topic, err := b.store.Event(t.Context(), roomID, topicID) + if err != nil { + t.Fatalf("Event(topic): %v", err) + } + for pos, wantTopic := range map[int64]bool{ + storage.PositionBefore(topic.StreamOrdering): false, + topic.StreamOrdering: true, + } { + state, err := b.api.StateAt(t.Context(), roomID, userB, pos) + if err != nil { + t.Fatalf("StateAt(%d): %v", pos, err) + } + hasTopic, hasName := false, false + for _, e := range state { + hasTopic = hasTopic || e.Type == roomserver.TypeTopic + hasName = hasName || e.Type == roomserver.TypeName + } + if hasTopic != wantTopic || !hasName { + t.Errorf("state at %d: topic %v, name %v; want topic %v and the name", pos, hasTopic, hasName, wantTopic) + } + } + + // Scrolling back again asks nobody anything and shows the same. + if again := b.scrollBack(t, roomID, userB, 7); !slices.Equal(again, want) { + t.Errorf("a second scrollback differs:\n%v\n%v", again, want) + } + }) + } +} + +func TestBackfillWithFederationOffShowsWhatIsHeld(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + for i := range 5 { + a.send(t, roomID, userA, fmt.Sprint(i)) + } + a.takeOutbox() + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{ + RoomID: roomID, User: userB, ServerNames: []string{serverA}, + }); err != nil { + t.Fatalf("remote join: %v", err) + } + + // With the switch off the page is what B has, and no error: the client + // is not told /messages failed because another server could not be + // asked. + m.setDisabled(true) + got := b.scrollBack(t, roomID, userB, 10) + if held := b.timelineIDs(t, roomID); !slices.Equal(got, held) { + t.Errorf("scrollback with federation off = %v, want what B holds: %v", got, held) + } + + // And once it is back on, the history is there. + m.setDisabled(false) + if got := b.scrollBack(t, roomID, userB, 10); !slices.Equal(got, a.timelineIDs(t, roomID)) { + t.Errorf("scrollback with federation back on = %d events, want A's %d", len(got), len(a.timelineIDs(t, roomID))) + } +} + +func TestBackfillIsNotAskedForUnderJoinedVisibility(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + if _, err := a.api.SendEvent(t.Context(), roomserver.SendEventRequest{ + RoomID: roomID, Sender: userA, Type: event.TypeHistoryVisibility, StateKey: ptrTo(""), + Content: []byte(`{"history_visibility":"joined"}`), + }); err != nil { + t.Fatalf("SendEvent: %v", err) + } + a.send(t, roomID, userA, "before") + a.takeOutbox() + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{ + RoomID: roomID, User: userB, ServerNames: []string{serverA}, + }); err != nil { + t.Fatalf("remote join: %v", err) + } + + // Bob may not see before his join, so there is nothing to fetch for + // him, and nothing is. + if got := b.scrollBack(t, roomID, userB, 10); len(got) != 1 { + t.Errorf("scrollback = %v, want just the join", got) + } + if held := b.timelineIDs(t, roomID); len(held) != 1 { + t.Errorf("B's timeline = %v, want nothing backfilled", held) + } +} diff --git a/server/internal/roomserver/directory.go b/server/internal/roomserver/directory.go index e7a5a8e..6ae0b48 100644 --- a/server/internal/roomserver/directory.go +++ b/server/internal/roomserver/directory.go @@ -155,13 +155,7 @@ func (r *API) ResolveAlias(ctx context.Context, alias string) (ResolvedAlias, er return ResolvedAlias{}, err } if !r.isLocalAlias(alias) { - if !r.federationEnabled() { - return ResolvedAlias{}, fmt.Errorf( - "%w: %s belongs to another server, and resolving a remote alias", - ErrFederationDisabled, alias) - } - return ResolvedAlias{}, fmt.Errorf( - "%w: resolving an alias of another server is not implemented yet", ErrFederationDisabled) + return r.resolveRemoteAlias(ctx, alias) } stored, err := r.store.Alias(ctx, alias) @@ -282,12 +276,7 @@ type PublicRoomsResponse struct { // the token says. func (r *API) PublicRooms(ctx context.Context, req PublicRoomsRequest) (PublicRoomsResponse, error) { if req.Server != "" && req.Server != r.serverName { - if !r.federationEnabled() { - return PublicRoomsResponse{}, fmt.Errorf( - "%w: reading the room directory of %s", ErrFederationDisabled, req.Server) - } - return PublicRoomsResponse{}, fmt.Errorf( - "%w: reading another server's room directory is not implemented yet", ErrFederationDisabled) + return r.remotePublicRooms(ctx, req) } offset, err := parseDirectoryToken(req.Since) diff --git a/server/internal/roomserver/federated.go b/server/internal/roomserver/federated.go new file mode 100644 index 0000000..078b894 --- /dev/null +++ b/server/internal/roomserver/federated.go @@ -0,0 +1,689 @@ +package roomserver + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sync" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/auth" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/matrix/stateres" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// The intake of events from other servers. +// +// An event created here is built on the room's current state and can only +// fail the authorisation rules. An event from somewhere else arrives with +// none of that settled: it was built on whatever its sender knew, its auth +// events and its prev_events may be events this server has never seen, +// and it may be perfectly valid where it was made and wrong for the room as +// this server knows it. The spec's procedure for it, which this file +// follows step by step, is: +// +// 1. It is well-formed, signed by its sender's server, and its content +// matches its hash — or it is replaced by its redacted form. +// 2. It passes the authorisation rules against its own auth_events, +// which are fetched if they are missing. +// 3. It passes them against the room's state *before* it — the state +// after its prev_events, resolved when there are several. A prev_event +// this server does not have is asked for with /get_missing_events and, +// if the gap is too wide, replaced by asking for the state at it. +// 4. It passes them against the room's *current* state. One that fails +// only here is soft-failed: kept, so that other events can cite it, but +// never shown to a client and never a forward extremity. +// +// Failing 1 to 3 is a rejection, and a rejected event is not stored. + +// Bounds on how much one event can make this server fetch. +const ( + // missingEventsLimit is how many events /get_missing_events is asked + // for. A gap wider than this is closed by fetching the state instead. + missingEventsLimit = 10 + + // stateFetchByIDLimit is how many missing state events are fetched one + // at a time with /event before it is cheaper to ask for the whole + // state with /state. + stateFetchByIDLimit = 50 + + // fetchConcurrency bounds the /event requests in flight at once. + fetchConcurrency = 8 +) + +// ReceivePDU takes one PDU from a /send transaction. It returns the event +// ID — which a transaction's response is keyed by — and the reason the +// event was refused, if it was. +func (r *API) ReceivePDU(ctx context.Context, origin string, raw json.RawMessage) (string, error) { + if err := r.requireFederation("receiving events from another server"); err != nil { + return "", err + } + roomID, err := roomIDOfRaw(raw) + if err != nil { + return "", err + } + room, rules, err := r.roomAndRules(ctx, roomID) + if errors.Is(err, ErrNoSuchRoom) { + return "", fmt.Errorf("%w: %s", ErrNotInRoom, roomID) + } else if err != nil { + return "", err + } + e, err := parseRemote(raw, rules) + if err != nil { + return "", err + } + if in, err := r.participating(ctx, roomID); err != nil { + return e.EventID(), err + } else if !in { + return e.EventID(), fmt.Errorf("%w: this server is not in %s", ErrNotInRoom, roomID) + } + if err := r.requireAllowed(ctx, roomID, origin); err != nil { + return e.EventID(), err + } + return e.EventID(), r.receive(ctx, incoming{origin: origin, room: room, rules: rules, event: e, fetchMissing: true}) +} + +// incoming is one event on its way in. +type incoming struct { + origin string + room storage.Room + rules roomversion.Rules + event *event.Event + + // fetchMissing allows /get_missing_events. It is off for the events + // that call itself returns, so that a gap is closed one level deep and + // no further. + fetchMissing bool + + // relay marks an event handed over by send_join or send_leave; see + // Notification.Relay. + relay bool + + // batch is the part of a backfill batch already accepted, which is + // not stored until the whole batch is (see backfill.go). The events in + // it are this event's likeliest auth and prev events, so they are + // looked for here before the store and before asking anybody. + batch map[string]batchEvent +} + +// receive runs the intake procedure. +func (r *API) receive(ctx context.Context, in incoming) error { + // An event already held, as an outlier or in the timeline, has been + // through this before. An outlier is left an outlier even when it + // arrives again as an ordinary event: it is one because this server + // used it as state without having its history, and making it a forward + // extremity now would put the room's head on an event whose children + // may already be here. + if _, err := r.store.Event(ctx, in.room.ID, in.event.EventID()); err == nil { + return nil + } else if !isNotFound(err) { + return err + } + + e, err := r.verifyIncoming(ctx, in.event) + if err != nil { + return err + } + in.event = e + + // Step 2: the event against its own auth events. + authEvents, err := r.authEventsOf(ctx, in) + if err != nil { + return err + } + create, err := r.createEvent(ctx, in.room, in.rules) + if err != nil { + return err + } + if err := auth.CheckAuthEvents(e, authEvents); err != nil { + return err + } + if err := auth.Check(ctx, e, withCreate(auth.NewState(authEvents), create), r.verifier()); err != nil { + return err + } + + // Step 3: the event against the state before it. + stateBefore, err := r.stateBeforeIncoming(ctx, in) + if err != nil { + return err + } + if err := auth.Check(ctx, e, withCreate(stateBefore, create), r.verifier()); err != nil { + return fmt.Errorf("%w (against the state before it)", err) + } + + return r.persistIncoming(ctx, in, stateBefore) +} + +// withCreate adds the create event to a state that lacks it, which from +// room version 12 is every state built from auth_events. +func withCreate(st auth.State, create *event.Event) auth.State { + if st.Create() == nil && create != nil { + st[auth.StateKey{Type: event.TypeCreate}] = create + } + return st +} + +// createEvent reads a room's create event. +func (r *API) createEvent(ctx context.Context, room storage.Room, rules roomversion.Rules) (*event.Event, error) { + stored, err := r.store.CurrentStateEvent(ctx, room.ID, event.TypeCreate, "") + if err != nil { + if isNotFound(err) { + return nil, fmt.Errorf("%w: this server does not hold the state of %s", ErrNotInRoom, room.ID) + } + return nil, err + } + return parseEvent(stored, rules) +} + +// persistIncoming is step 4 and the write: under the room lock, the event +// is checked against the current state and either soft-failed or added to +// the room's head. +func (r *API) persistIncoming(ctx context.Context, in incoming, stateBefore auth.State) error { + e := in.event + mu := r.lock(in.room.ID) + mu.Lock() + defer mu.Unlock() + + // Another transaction may have delivered the same event while this one + // was fetching what it needed. + if _, err := r.store.Event(ctx, in.room.ID, e.EventID()); err == nil { + return nil + } else if !isNotFound(err) { + return err + } + + current, extremities, _, err := r.roomState(ctx, in.room, in.rules) + if err != nil { + return err + } + + // Step 4: soft failure. + if err := auth.Check(ctx, e, current, r.verifier()); err != nil { + r.log.Info("soft-failed an event from another server", + "room_id", in.room.ID, "event_id", e.EventID(), "type", e.Type(), + "sender", e.Sender(), "origin", in.origin, "reason", err) + return r.store.WriteOutliers(ctx, in.room.ID, []storage.StoredEvent{storedOf(in.room.ID, e)}) + } + + // The new forward extremities are the old ones this event does not + // descend from, plus the event. + prevs := e.PrevEventIDs() + var newExtremities []string + for _, x := range extremities { + if !slices.Contains(prevs, x.EventID()) { + newExtremities = append(newExtremities, x.EventID()) + } + } + newExtremities = append(newExtremities, e.EventID()) + + req := persistRequest{ + Room: in.room, + Rules: in.rules, + Event: e, + StateBefore: stateBefore, + Extremities: newExtremities, + Origin: in.origin, + Relay: in.relay, + } + if e.Type() == event.TypeRedaction { + req.RedactsEventID = e.Redacts() + } + + // The ordinary case: the event follows the room's single extremity, so + // the state before it is the current state and the state after it is + // one tuple more. Anything else — a fork, or an event built on an + // older part of the room — means working the current state out again + // from the room's new head. + linear := len(extremities) == 1 && len(prevs) == 1 && prevs[0] == extremities[0].EventID() + if !linear { + resolved, err := r.stateAtHead(ctx, in.room, in.rules, newExtremities, e, stateBefore) + if err != nil { + return err + } + req.CurrentState = resolved + } + return r.persist(ctx, req) +} + +// stateAtHead resolves the room's state across a set of forward +// extremities, one of which is the event being persisted. +func (r *API) stateAtHead(ctx context.Context, room storage.Room, rules roomversion.Rules, extremityIDs []string, e *event.Event, stateBefore auth.State) (auth.State, error) { + var sets [][]*event.Event + for _, id := range extremityIDs { + if id == e.EventID() { + set := stateres.Sorted(stateBefore) + if e.IsState() { + set = append(set, e) + } + sets = append(sets, set) + continue + } + stored, err := r.store.Event(ctx, room.ID, id) + if err != nil { + return nil, err + } + x, err := parseEvent(stored, rules) + if err != nil { + return nil, err + } + set, err := r.stateAfter(ctx, room.ID, rules, x) + if err != nil { + return nil, err + } + sets = append(sets, set) + } + if len(sets) == 1 { + return auth.NewState(sets[0]), nil + } + resolved, err := stateres.Resolve(ctx, rules, sets, eventProvider{store: r.store, roomID: room.ID, rules: rules}) + if err != nil { + return nil, fmt.Errorf("resolving the state of %s: %w", room.ID, err) + } + return resolved, nil +} + +// authEventsOf returns an incoming event's auth events, fetching the ones +// this server does not hold. +func (r *API) authEventsOf(ctx context.Context, in incoming) ([]*event.Event, error) { + ids := in.event.AuthEventIDs() + have, err := r.heldOrBatched(ctx, in, ids) + if err != nil { + return nil, err + } + if len(have) < len(ids) { + if err := r.fetchAuthChain(ctx, in); err != nil { + return nil, err + } + if have, err = r.heldOrBatched(ctx, in, ids); err != nil { + return nil, err + } + } + if len(have) < len(ids) { + return nil, fmt.Errorf("%w: %s cites auth events that %s could not supply", + ErrBadEvent, in.event.EventID(), in.origin) + } + + // Keep the order the event cites them in; CheckAuthEvents does not + // care, but a log line naming them does. + byID := map[string]*event.Event{} + for _, e := range have { + byID[e.EventID()] = e + } + out := make([]*event.Event, 0, len(ids)) + for _, id := range ids { + out = append(out, byID[id]) + } + return out, nil +} + +// heldOrBatched is eventsByID that also finds the events of the backfill +// batch an incoming event belongs to. +func (r *API) heldOrBatched(ctx context.Context, in incoming, ids []string) ([]*event.Event, error) { + var out, rest []string + for _, id := range ids { + if _, ok := in.batch[id]; ok { + out = append(out, id) + } else { + rest = append(rest, id) + } + } + held, err := r.eventsByID(ctx, in.room.ID, in.rules, rest) + if err != nil { + return nil, err + } + for _, id := range out { + held = append(held, in.batch[id].event) + } + return held, nil +} + +// eventsByID reads and parses the events of a room this server holds, +// outliers included, leaving out the ones it does not. +func (r *API) eventsByID(ctx context.Context, roomID string, rules roomversion.Rules, ids []string) ([]*event.Event, error) { + if len(ids) == 0 { + return nil, nil + } + stored, err := r.store.EventsByID(ctx, roomID, ids) + if err != nil { + return nil, err + } + return parseEvents(stored, rules) +} + +// fetchAuthChain asks the origin for an event's whole auth chain and +// stores what passes as outliers. +func (r *API) fetchAuthChain(ctx context.Context, in incoming) error { + resp, err := r.remote.EventAuth(ctx, in.origin, in.room.ID, in.event.EventID()) + if err != nil { + return remoteError(in.origin, err) + } + return r.acceptOutliers(ctx, in.room, in.rules, resp.AuthChain) +} + +// acceptOutliers parses, verifies and authorises a batch of events from +// another server, and stores the ones that pass as outliers. +func (r *API) acceptOutliers(ctx context.Context, room storage.Room, rules roomversion.Rules, raws []json.RawMessage) error { + var fresh []*event.Event + referenced := map[string]bool{} + ids := make([]string, 0, len(raws)) + for _, raw := range raws { + e, err := parseRemote(raw, rules) + if err != nil { + r.log.Debug("ignoring an unparseable event", "room_id", room.ID, "error", err) + continue + } + if !inRoom(e, room.ID) { + continue + } + ids = append(ids, e.EventID()) + fresh = append(fresh, e) + for _, id := range e.AuthEventIDs() { + referenced[id] = true + } + } + + // What is already here does not need fetching, verifying or storing + // again; it is only needed as the auth events of what is new. + held, err := r.eventsByID(ctx, room.ID, rules, ids) + if err != nil { + return err + } + heldIDs := map[string]bool{} + for _, e := range held { + heldIDs[e.EventID()] = true + } + var toCheck []*event.Event + for _, e := range fresh { + if !heldIDs[e.EventID()] { + toCheck = append(toCheck, e) + } + } + if len(toCheck) == 0 { + return nil + } + + verified := r.verifyAll(ctx, toCheck) + + knownIDs := make([]string, 0, len(referenced)) + for id := range referenced { + knownIDs = append(knownIDs, id) + } + knownEvents, err := r.eventsByID(ctx, room.ID, rules, knownIDs) + if err != nil { + return err + } + known := map[string]*event.Event{} + for _, e := range knownEvents { + known[e.EventID()] = e + } + if create, err := r.createEvent(ctx, room, rules); err == nil { + known[create.EventID()] = create + } + + passed, rejected := r.authCheckChain(ctx, verified, known) + if len(rejected) > 0 { + r.log.Info("rejected events in a fetched auth chain or state", + "room_id", room.ID, "rejected", len(rejected)) + } + rows := make([]storage.StoredEvent, 0, len(passed)) + for _, e := range verified { + if _, ok := passed[e.EventID()]; ok { + rows = append(rows, storedOf(room.ID, e)) + } + } + return r.store.WriteOutliers(ctx, room.ID, rows) +} + +// verifyAll checks the signatures and hashes of a batch, in parallel, and +// returns the events that pass in their original order. +func (r *API) verifyAll(ctx context.Context, events []*event.Event) []*event.Event { + out := make([]*event.Event, len(events)) + sem := make(chan struct{}, fetchConcurrency) + var wg sync.WaitGroup + for i, e := range events { + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + verified, err := r.verifyIncoming(ctx, e) + if err != nil { + r.log.Debug("dropping an event that does not verify", "event_id", e.EventID(), "error", err) + return + } + out[i] = verified + }() + } + wg.Wait() + return slices.DeleteFunc(out, func(e *event.Event) bool { return e == nil }) +} + +// stateBeforeIncoming works out the room state an incoming event was built +// on: the state after each of its prev_events, resolved together. +func (r *API) stateBeforeIncoming(ctx context.Context, in incoming) (auth.State, error) { + prevs := in.event.PrevEventIDs() + if len(prevs) == 0 { + return nil, fmt.Errorf("%w: %s has no prev_events", ErrBadEvent, in.event.EventID()) + } + + // A prev_event earlier in the same backfill batch has its state after + // it in hand already. + var sets [][]*event.Event + var stored []string + for _, id := range prevs { + if b, ok := in.batch[id]; ok { + sets = append(sets, b.stateAfter) + } else { + stored = append(stored, id) + } + } + + known, missing, err := r.splitPrevs(ctx, in.room, in.rules, stored) + if err != nil { + return nil, err + } + if len(missing) > 0 && in.fetchMissing { + r.fetchMissingEvents(ctx, in) + if known, missing, err = r.splitPrevs(ctx, in.room, in.rules, stored); err != nil { + return nil, err + } + } + + for _, prev := range known { + set, err := r.stateAfter(ctx, in.room.ID, in.rules, prev) + if err != nil { + return nil, err + } + sets = append(sets, set) + } + for _, prevID := range missing { + set, err := r.stateAfterRemote(ctx, in, prevID) + if err != nil { + return nil, err + } + sets = append(sets, set) + } + + if len(sets) == 1 { + return auth.NewState(sets[0]), nil + } + resolved, err := stateres.Resolve(ctx, in.rules, sets, + eventProvider{store: r.store, roomID: in.room.ID, rules: in.rules}) + if err != nil { + return nil, fmt.Errorf("resolving the state before %s: %w", in.event.EventID(), err) + } + return resolved, nil +} + +// splitPrevs sorts prev_events into the ones in this server's timeline, +// which have a state snapshot, and the ones it lacks. An outlier counts as +// lacking: it is held, but the state at it is not. +func (r *API) splitPrevs(ctx context.Context, room storage.Room, rules roomversion.Rules, prevs []string) ([]*event.Event, []string, error) { + stored, err := r.store.EventsByID(ctx, room.ID, prevs) + if err != nil { + return nil, nil, err + } + have := map[string]storage.StoredEvent{} + for _, s := range stored { + if !s.Outlier { + have[s.EventID] = s + } + } + var known []*event.Event + var missing []string + for _, id := range prevs { + s, ok := have[id] + if !ok { + missing = append(missing, id) + continue + } + e, err := parseEvent(s, rules) + if err != nil { + return nil, nil, err + } + known = append(known, e) + } + return known, missing, nil +} + +// fetchMissingEvents asks the origin for the events between this server's +// head of the room and the incoming event, and takes each of them in, +// oldest first. Failures are logged, not returned: whatever this closes of +// the gap helps, and what it leaves open is closed by fetching state. +func (r *API) fetchMissingEvents(ctx context.Context, in incoming) { + stored, err := r.store.Extremities(ctx, in.room.ID) + if err != nil { + r.log.Warn("could not read extremities to fetch missing events", "room_id", in.room.ID, "error", err) + return + } + earliest := make([]string, 0, len(stored)) + var minDepth int64 + for i, s := range stored { + earliest = append(earliest, s.EventID) + if i == 0 || s.Depth < minDepth { + minDepth = s.Depth + } + } + + resp, err := r.remote.MissingEvents(ctx, in.origin, in.room.ID, federation.MissingEventsRequest{ + EarliestEvents: earliest, + LatestEvents: []string{in.event.EventID()}, + Limit: missingEventsLimit, + MinDepth: minDepth, + }) + if err != nil { + r.log.Info("could not fetch missing events", "room_id", in.room.ID, + "origin", in.origin, "event_id", in.event.EventID(), "error", err) + return + } + + var events []*event.Event + for _, raw := range resp.Events { + e, err := parseRemote(raw, in.rules) + if err != nil || e.RoomID() != in.room.ID || e.EventID() == in.event.EventID() { + continue + } + events = append(events, e) + } + eventsByDepth(events) + for _, e := range events { + err := r.receive(ctx, incoming{origin: in.origin, room: in.room, rules: in.rules, event: e}) + if err != nil { + r.log.Info("a fetched missing event was not accepted", + "room_id", in.room.ID, "event_id", e.EventID(), "error", err) + } + } +} + +// stateAfterRemote asks the origin for the state at a prev_event this +// server does not have, fetches whatever of it is missing, and returns the +// state after that event. +// +// /state_ids answers with the state *before* the event it is asked about, +// so the event itself is fetched too and, if it is a state event, added on +// top. Everything fetched is authorised against its own auth events and +// stored as an outlier: it is real, but this server has none of the +// history that would put it in the timeline. +func (r *API) stateAfterRemote(ctx context.Context, in incoming, prevID string) ([]*event.Event, error) { + ids, err := r.remote.StateIDs(ctx, in.origin, in.room.ID, prevID) + if err != nil { + return nil, remoteError(in.origin, err) + } + + need := append(append(slices.Clone(ids.PDUIDs), ids.AuthChainIDs...), prevID) + have, err := r.store.EventsByID(ctx, in.room.ID, need) + if err != nil { + return nil, err + } + haveIDs := map[string]bool{} + for _, s := range have { + haveIDs[s.EventID] = true + } + var missing []string + for _, id := range need { + if !haveIDs[id] && !slices.Contains(missing, id) { + missing = append(missing, id) + } + } + + if len(missing) > 0 { + var raws []json.RawMessage + if len(missing) > stateFetchByIDLimit { + state, err := r.remote.State(ctx, in.origin, in.room.ID, prevID) + if err != nil { + return nil, remoteError(in.origin, err) + } + raws = append(append(raws, state.AuthChain...), state.PDUs...) + if !slices.Contains(missing, prevID) { + missing = nil + } else { + missing = []string{prevID} + } + } + raws = append(raws, r.fetchEvents(ctx, in.origin, missing)...) + if err := r.acceptOutliers(ctx, in.room, in.rules, raws); err != nil { + return nil, err + } + } + + state, err := r.eventsByID(ctx, in.room.ID, in.rules, ids.PDUIDs) + if err != nil { + return nil, err + } + prev, err := r.eventsByID(ctx, in.room.ID, in.rules, []string{prevID}) + if err != nil { + return nil, err + } + if len(prev) == 1 && prev[0].IsState() { + state = append(state, prev[0]) + } + return state, nil +} + +// fetchEvents fetches events one by one from a server, in parallel, +// returning the ones it supplied. +func (r *API) fetchEvents(ctx context.Context, server string, ids []string) []json.RawMessage { + results := make([]json.RawMessage, len(ids)) + sem := make(chan struct{}, fetchConcurrency) + var wg sync.WaitGroup + for i, id := range ids { + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + txn, err := r.remote.Event(ctx, server, id) + if err != nil || len(txn.PDUs) == 0 { + r.log.Debug("could not fetch an event", "server", server, "event_id", id, "error", err) + return + } + results[i] = txn.PDUs[0] + }() + } + wg.Wait() + return slices.DeleteFunc(results, func(raw json.RawMessage) bool { return raw == nil }) +} diff --git a/server/internal/roomserver/federated_directory.go b/server/internal/roomserver/federated_directory.go new file mode 100644 index 0000000..2f659f6 --- /dev/null +++ b/server/internal/roomserver/federated_directory.go @@ -0,0 +1,53 @@ +package roomserver + +import ( + "context" + "encoding/json" + "fmt" + "strings" +) + +// resolveRemoteAlias asks the server an alias belongs to what it points +// at. Remote aliases are never cached: the owning server may move one at +// any time, and a stale answer would send a join to the wrong room. +func (r *API) resolveRemoteAlias(ctx context.Context, alias string) (ResolvedAlias, error) { + if err := r.requireFederation(fmt.Sprintf("%s belongs to another server, and resolving a remote alias", alias)); err != nil { + return ResolvedAlias{}, err + } + _, server, _ := strings.Cut(alias[1:], ":") + resp, err := r.remote.QueryDirectory(ctx, server, alias) + if err != nil { + err = remoteError(server, err) + if isNotFound(err) { + return ResolvedAlias{}, fmt.Errorf("%w: %s", ErrNoSuchAlias, alias) + } + return ResolvedAlias{}, err + } + if resp.RoomID == "" { + return ResolvedAlias{}, fmt.Errorf("%w: %s answered for %s without a room ID", ErrRemote, server, alias) + } + servers := resp.Servers + if len(servers) == 0 { + servers = []string{server} + } + return ResolvedAlias{RoomID: resp.RoomID, Servers: servers}, nil +} + +// remotePublicRooms reads another server's public room directory. +func (r *API) remotePublicRooms(ctx context.Context, req PublicRoomsRequest) (PublicRoomsResponse, error) { + if err := r.requireFederation("reading the room directory of " + req.Server); err != nil { + return PublicRoomsResponse{}, err + } + raw, err := r.remote.PublicRooms(ctx, req.Server, req.Limit, req.Since, req.Search) + if err != nil { + return PublicRoomsResponse{}, remoteError(req.Server, err) + } + var resp PublicRoomsResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return PublicRoomsResponse{}, fmt.Errorf("%w: %s sent an unreadable directory: %v", ErrRemote, req.Server, err) + } + if resp.Chunk == nil { + resp.Chunk = []PublicRoom{} + } + return resp, nil +} diff --git a/server/internal/roomserver/federated_membership.go b/server/internal/roomserver/federated_membership.go new file mode 100644 index 0000000..9e3bb21 --- /dev/null +++ b/server/internal/roomserver/federated_membership.go @@ -0,0 +1,787 @@ +package roomserver + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/auth" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" + "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Membership across servers. +// +// A server can only sign as itself, and a room's state lives with the +// servers in it, so every membership change that crosses a server +// boundary is a little protocol of its own: +// +// - Joining a room this server is not in: ask a resident server for a +// join template (make_join), fill it in and sign it, and hand it back +// (send_join), which returns the room's state. +// - Leaving one it is not in — rejecting an invite, in practice — is the +// same with make_leave and send_leave. +// - Inviting a user of another server: their server has to countersign +// the invite before it can go into the room. +// +// And the other side of each, for when the other server is the one asking. + +// joinNeedsFederation reports whether a join has to go through another +// server: this server does not hold the room, holds only an invite to it, +// or holds a stale copy of a room it has left that still has remote +// members to ask. +func (r *API) joinNeedsFederation(ctx context.Context, req JoinRequest) (bool, error) { + if _, err := r.store.Room(ctx, req.RoomID); errors.Is(err, ErrNoSuchRoom) { + return true, nil + } else if err != nil { + return false, err + } + if _, err := r.store.CurrentStateEvent(ctx, req.RoomID, event.TypeCreate, ""); isNotFound(err) { + return true, nil + } else if err != nil { + return false, err + } + in, err := r.participating(ctx, req.RoomID) + if err != nil || in { + return false, err + } + // Every local user has left. A room that only ever had local members + // is still this server's to rejoin on its own; one with members + // elsewhere has moved on without us, and has to be asked. + servers, err := r.JoinedServers(ctx, req.RoomID) + if err != nil { + return false, err + } + return slices.ContainsFunc(servers, func(s string) bool { return s != r.serverName }), nil +} + +// joinRemote joins a room through another server, trying each candidate +// in turn. +func (r *API) joinRemote(ctx context.Context, req JoinRequest) (string, error) { + if err := r.requireFederation(fmt.Sprintf("%s is not a room on this server, and joining a remote room", req.RoomID)); err != nil { + return "", err + } + servers := r.candidateServers(ctx, req.RoomID, req.ServerNames) + if len(servers) == 0 { + return "", fmt.Errorf("%w: no server to join %s through", ErrNoSuchRoom, req.RoomID) + } + + var firstErr error + for _, server := range servers { + err := r.joinVia(ctx, server, req) + if err == nil { + return req.RoomID, nil + } + r.log.Info("joining through a server failed", + "room_id", req.RoomID, "server", server, "user_id", req.User, "error", err) + // The first server's answer is the one worth reporting: it is the + // one the client named, or the room's own. + if firstErr == nil { + firstErr = err + } + } + return "", firstErr +} + +// eventTemplate is the part of a make_join or make_leave answer the +// joining server uses. +type eventTemplate struct { + RoomID string `json:"room_id"` + Type string `json:"type"` + StateKey *string `json:"state_key"` + Sender string `json:"sender"` + Content json.RawMessage `json:"content"` + PrevEvents []string `json:"prev_events"` + AuthEvents []string `json:"auth_events"` + Depth int64 `json:"depth"` +} + +// buildFromTemplate fills in and signs a membership template from another +// server. The template's DAG references are kept as given — they are what +// the resident server knows and this one does not — and its content is +// merged with this server's own, so that a restricted join keeps the +// authorisation the resident server chose. +func (r *API) buildFromTemplate(raw json.RawMessage, rules roomversion.Rules, roomID, userID, membership string, extra map[string]any) (*event.Event, error) { + var tmpl eventTemplate + if err := json.Unmarshal(raw, &tmpl); err != nil { + return nil, fmt.Errorf("%w: the event template is unreadable: %v", ErrRemote, err) + } + if tmpl.Type != event.TypeMember || tmpl.StateKey == nil || *tmpl.StateKey != userID || + tmpl.Sender != userID || tmpl.RoomID != roomID { + return nil, fmt.Errorf("%w: the event template is not a membership event for %s in %s", ErrRemote, userID, roomID) + } + + content := map[string]any{} + if len(tmpl.Content) > 0 { + if err := json.Unmarshal(tmpl.Content, &content); err != nil { + return nil, fmt.Errorf("%w: the template content is not an object", ErrRemote) + } + } + for k, v := range extra { + content[k] = v + } + content["membership"] = membership + + builder := event.Builder{ + RoomID: roomID, + Type: event.TypeMember, + StateKey: statePtr(userID), + Sender: userID, + Content: mustMarshal(content), + PrevEvents: tmpl.PrevEvents, + AuthEvents: tmpl.AuthEvents, + Depth: tmpl.Depth, + } + built, err := builder.Build(r.now(), r.signingKey, rules) + if err != nil { + return nil, fmt.Errorf("%w: the template does not make a valid event: %v", ErrRemote, err) + } + return built, nil +} + +// joinVia runs make_join and send_join against one server. +func (r *API) joinVia(ctx context.Context, server string, req JoinRequest) error { + tmpl, err := r.remote.MakeJoin(ctx, server, req.RoomID, req.User, supportedVersionStrings()) + if err != nil { + return remoteError(server, err) + } + version := tmpl.RoomVersion + if version == "" { + // The spec's default for a server too old to say. + version = "1" + } + rules, err := roomversion.Get(roomversion.Version(version)) + if err != nil { + return fmt.Errorf("%w: %s is version %q: %v", ErrIncompatibleRoomVersion, req.RoomID, version, err) + } + + // The profile and the reason are this server's to add; anything about + // authorisation is the template's. + extra := r.membershipContentMap(ctx, event.MembershipJoin, req.User, req.Reason, false) + delete(extra, "membership") + join, err := r.buildFromTemplate(tmpl.Event, rules, req.RoomID, req.User, event.MembershipJoin, extra) + if err != nil { + return err + } + + resp, err := r.remote.SendJoin(ctx, server, req.RoomID, join.EventID(), join.JSON()) + if err != nil { + return remoteError(server, err) + } + // A restricted join comes back countersigned by the server that + // authorised it, and that copy is the one the room will hold. + if len(resp.Event) > 0 { + countersigned, err := parseRemote(resp.Event, rules) + if err == nil && countersigned.EventID() == join.EventID() { + join = countersigned + } + } + return r.completeJoin(ctx, server, req.RoomID, rules, join, resp) +} + +// completeJoin takes in the room a send_join answer describes, and the +// join on top of it. +// +// Every event of the state and auth chain is verified and authorised +// against its own auth events before any of it is believed: the resident +// server is trusted to know the room, not to tell the truth about it. +func (r *API) completeJoin(ctx context.Context, server, roomID string, rules roomversion.Rules, join *event.Event, resp federation.SendJoinResponse) error { + byID := map[string]*event.Event{} + var all []*event.Event + stateIDs := map[string]bool{} + for i, raw := range append(slices.Clone(resp.State), resp.AuthChain...) { + e, err := parseRemote(raw, rules) + if err != nil || !inRoom(e, roomID) { + continue + } + if i < len(resp.State) { + stateIDs[e.EventID()] = true + } + if _, dup := byID[e.EventID()]; dup { + continue + } + byID[e.EventID()] = e + all = append(all, e) + } + + verified := r.verifyAll(ctx, all) + passed, rejected := r.authCheckChain(ctx, verified, nil) + if len(rejected) > 0 { + r.log.Info("rejected events in a send_join answer", + "room_id", roomID, "server", server, "rejected", len(rejected)) + } + + var stateEvents []*event.Event + for _, e := range verified { + if stateIDs[e.EventID()] && e.IsState() { + if _, ok := passed[e.EventID()]; ok { + stateEvents = append(stateEvents, e) + } + } + } + state := auth.NewState(stateEvents) + create := state.Create() + if create == nil { + return fmt.Errorf("%w: %s sent the state of %s without its create event", ErrRemote, server, roomID) + } + if err := auth.Check(ctx, join, state, r.verifier()); err != nil { + return fmt.Errorf("the room as %s describes it does not allow the join: %w", server, err) + } + + mu := r.lock(roomID) + mu.Lock() + defer mu.Unlock() + + if err := r.store.CreateRoom(ctx, storage.NewRoom{ + ID: roomID, + Version: string(rules.Version), + Creator: create.Sender(), + CreatedAtMS: r.nowMS(), + }); err != nil && !errors.Is(err, storage.ErrRoomExists) { + return err + } + room, err := r.store.Room(ctx, roomID) + if err != nil { + return err + } + if room.Version != string(rules.Version) { + return fmt.Errorf("%w: this server knows %s as version %s, and %s says %s", + ErrIncompatibleRoomVersion, roomID, room.Version, server, rules.Version) + } + + rows := make([]storage.StoredEvent, 0, len(passed)) + for _, e := range verified { + if _, ok := passed[e.EventID()]; ok { + rows = append(rows, storedOf(roomID, e)) + } + } + if err := r.store.WriteOutliers(ctx, roomID, rows); err != nil { + return err + } + + stateKey, _ := join.StateKey() + if err := r.persist(ctx, persistRequest{ + Room: room, + Rules: rules, + Event: join, + StateBefore: state, + CurrentState: withState(state, event.TypeMember, stateKey, join), + // The resident server has the join already and passes it on to + // everybody else; sending it from here as well would only double + // the traffic. + Origin: server, + }); err != nil { + return err + } + r.log.Info("joined a room over federation", + "room_id", roomID, "via", server, "user_id", join.Sender(), + "state_events", len(stateEvents), "auth_chain", len(resp.AuthChain)) + return nil +} + +// leaveRemote leaves a room this server is not in — rejects an invite, +// that is — through a server that is. +// +// If no server can be reached the invite is still rejected locally: a +// leave is built and stored here and nowhere else. The inviting server +// never hears of it, which is the lesser problem; an invite the user +// cannot get rid of because its server is gone is the greater one. +func (r *API) leaveRemote(ctx context.Context, roomID, userID, reason string) error { + room, rules, err := r.roomAndRules(ctx, roomID) + if err != nil { + return err + } + membership, err := r.store.Membership(ctx, roomID, userID) + if err != nil { + return err + } + if membership.Membership != event.MembershipInvite && membership.Membership != event.MembershipKnock { + return fmt.Errorf("%w: %s is not in %s", ErrForbidden, userID, roomID) + } + + extra := map[string]any{} + if reason != "" { + extra["reason"] = reason + } + if r.federating() { + for _, server := range r.candidateServers(ctx, roomID, nil) { + leave, err := r.leaveVia(ctx, server, rules, roomID, userID, extra) + if err != nil { + r.log.Info("leaving through a server failed", + "room_id", roomID, "server", server, "user_id", userID, "error", err) + continue + } + return r.persistOutOfBand(ctx, room, leave, nil, server) + } + } + + // Nobody could be asked: reject locally, on top of the invite. + inviteStored, err := r.store.Event(ctx, roomID, membership.EventID) + if err != nil { + return err + } + invite, err := parseEvent(inviteStored, rules) + if err != nil { + return err + } + extra["membership"] = event.MembershipLeave + builder := event.Builder{ + RoomID: roomID, + Type: event.TypeMember, + StateKey: statePtr(userID), + Sender: userID, + Content: mustMarshal(extra), + PrevEvents: []string{invite.EventID()}, + AuthEvents: []string{invite.EventID()}, + Depth: nextDepth([]*event.Event{invite}), + } + leave, err := builder.Build(r.now(), r.signingKey, rules) + if err != nil { + return err + } + r.log.Info("rejected an invite locally; no server in the room could be reached", + "room_id", roomID, "user_id", userID) + return r.persistOutOfBand(ctx, room, leave, nil, "") +} + +// leaveVia runs make_leave and send_leave against one server. +func (r *API) leaveVia(ctx context.Context, server string, rules roomversion.Rules, roomID, userID string, extra map[string]any) (*event.Event, error) { + tmpl, err := r.remote.MakeLeave(ctx, server, roomID, userID) + if err != nil { + return nil, remoteError(server, err) + } + leave, err := r.buildFromTemplate(tmpl.Event, rules, roomID, userID, event.MembershipLeave, extra) + if err != nil { + return nil, err + } + if err := r.remote.SendLeave(ctx, server, roomID, leave.EventID(), leave.JSON()); err != nil { + return nil, remoteError(server, err) + } + return leave, nil +} + +// persistOutOfBand stores a membership event for a room this server is not +// in: an invite it was asked to countersign, or the rejection of one. +// +// The event is an outlier — there is no state here to put it in the +// timeline against — and still becomes the user's membership in the +// room's current state, which is the one thing /sync needs to tell them +// they were invited, or are no longer. +func (r *API) persistOutOfBand(ctx context.Context, room storage.Room, e *event.Event, inviteState []byte, origin string) error { + mu := r.lock(room.ID) + mu.Lock() + defer mu.Unlock() + + stateKey, _ := e.StateKey() + membership, err := e.Membership() + if err != nil { + return err + } + stored := storedOf(room.ID, e) + ordering, err := r.store.Write(ctx, storage.RoomWrite{ + Event: stored, + Outlier: true, + StateDelta: []storage.StateEntry{{Type: event.TypeMember, StateKey: stateKey, EventID: e.EventID(), Membership: membership}}, + InviteState: inviteState, + }) + if err != nil { + return err + } + if r.notifier != nil { + r.notifier.EventPersisted(ctx, Notification{ + RoomID: room.ID, + Event: e, + StreamOrdering: ordering, + StateChanged: true, + Origin: origin, + }) + } + return nil +} + +// inviteRemote sends an invite to a user of another server through the +// pipeline, with their server's countersignature taken under the room +// lock. +func (r *API) inviteRemote(ctx context.Context, req InviteRequest) error { + if err := r.requireFederation(fmt.Sprintf("%s is on another server, and inviting a remote user", req.Target)); err != nil { + return err + } + targetServer, err := id.ServerNameOf(req.Target) + if err != nil { + return fmt.Errorf("%w: %v", ErrInvalidContent, err) + } + room, rules, err := r.roomAndRules(ctx, req.RoomID) + if err != nil { + return err + } + + _, err = r.send(ctx, sendRequest{ + RoomID: req.RoomID, + Type: event.TypeMember, + StateKey: statePtr(req.Target), + Sender: req.Sender, + Content: r.membershipContent(ctx, event.MembershipInvite, req.Target, req.Reason, req.IsDirect), + countersign: func(ctx context.Context, invite *event.Event) (*event.Event, error) { + stripped, err := r.strippedState(ctx, room.ID, rules, []string{req.Sender}) + if err != nil { + return nil, err + } + strippedRaw := make([]json.RawMessage, 0, len(stripped)) + for _, s := range stripped { + strippedRaw = append(strippedRaw, mustMarshal(s)) + } + resp, err := r.remote.Invite(ctx, targetServer, room.ID, invite.EventID(), federation.InviteRequest{ + RoomVersion: string(rules.Version), + Event: invite.JSON(), + InviteRoomState: strippedRaw, + }) + if err != nil { + return nil, remoteError(targetServer, err) + } + signed, err := parseRemote(resp.Event, rules) + if err != nil { + return nil, fmt.Errorf("%w: %s countersigned something unreadable: %v", ErrRemote, targetServer, err) + } + if signed.EventID() != invite.EventID() { + return nil, fmt.Errorf("%w: %s returned a different event from the invite", ErrRemote, targetServer) + } + if err := r.verifier().VerifyEventSignature(ctx, targetServer, signed); err != nil { + return nil, fmt.Errorf("%w: %s did not countersign the invite: %v", ErrRemote, targetServer, err) + } + return signed, nil + }, + }) + return err +} + +// --------------------------------------------------------------------------- +// The other side: requests from servers joining, leaving and inviting. + +// MakeJoin answers GET /make_join: a join template for a user of origin. +func (r *API) MakeJoin(ctx context.Context, origin, roomID, userID string, versions []string) (string, json.RawMessage, error) { + return r.makeMembership(ctx, origin, roomID, userID, event.MembershipJoin, versions) +} + +// MakeLeave answers GET /make_leave. +func (r *API) MakeLeave(ctx context.Context, origin, roomID, userID string) (string, json.RawMessage, error) { + return r.makeMembership(ctx, origin, roomID, userID, event.MembershipLeave, nil) +} + +// makeMembership builds a membership template for another server's user. +// +// The template is the event as this server would build it, minus the +// signature and hashes only the user's own server can add. It is checked +// against the room before it is handed out, so that a banned user is told +// so at make_join rather than after signing and sending a join that was +// never going to be accepted. +func (r *API) makeMembership(ctx context.Context, origin, roomID, userID, membership string, versions []string) (string, json.RawMessage, error) { + if err := r.requireFederation("answering another server's membership request"); err != nil { + return "", nil, err + } + if server, err := id.ServerNameOf(userID); err != nil || server != origin { + return "", nil, fmt.Errorf("%w: %s cannot ask on behalf of %s", ErrForbidden, origin, userID) + } + room, rules, err := r.roomAndRules(ctx, roomID) + if err != nil { + return "", nil, err + } + if in, err := r.participating(ctx, roomID); err != nil { + return "", nil, err + } else if !in { + return "", nil, fmt.Errorf("%w: this server is not in %s", ErrNotInRoom, roomID) + } + if err := r.requireAllowed(ctx, roomID, origin); err != nil { + return "", nil, err + } + if membership == event.MembershipJoin { + if len(versions) == 0 { + versions = []string{"1"} + } + if !slices.Contains(versions, room.Version) { + return "", nil, fmt.Errorf("%w: %s is version %s", ErrIncompatibleRoomVersion, roomID, room.Version) + } + } + + mu := r.lock(roomID) + mu.Lock() + defer mu.Unlock() + + st, extremities, _, err := r.roomState(ctx, room, rules) + if err != nil { + return "", nil, err + } + content := map[string]any{"membership": membership} + if membership == event.MembershipJoin { + authoriser, err := r.restrictedJoinAuthoriser(ctx, JoinRequest{RoomID: roomID, User: userID}) + if err != nil { + return "", nil, err + } + if authoriser != "" { + content["join_authorised_via_users_server"] = authoriser + } + } + contentRaw := mustMarshal(content) + + authEvents, err := auth.SelectAuthEvents(rules, event.TypeMember, userID, &userID, contentRaw, st) + if err != nil { + return "", nil, fmt.Errorf("%w: %s", ErrRejected, err) + } + + tmpl := map[string]any{ + "room_id": roomID, + "type": event.TypeMember, + "state_key": userID, + "sender": userID, + "content": contentRaw, + "prev_events": eventIDs(extremities), + "auth_events": eventIDs(authEvents), + "depth": nextDepth(extremities), + "origin_server_ts": r.nowMS(), + "origin": r.serverName, + } + + // A trial build, signed with this server's key in the user's name. The + // signature is worthless and nothing here checks it: what is being + // asked is only whether the room would let this user in or out. + trial, err := (&event.Builder{ + RoomID: roomID, Type: event.TypeMember, StateKey: &userID, Sender: userID, + Content: contentRaw, PrevEvents: eventIDs(extremities), AuthEvents: eventIDs(authEvents), + Depth: nextDepth(extremities), + }).Build(r.now(), r.signingKey, rules) + if err != nil { + return "", nil, err + } + if _, ok := content["join_authorised_via_users_server"]; ok { + // The real join will carry this server's countersignature, added + // at send_join; the trial needs it too, or the rule that checks + // for it refuses the join this server is about to authorise. + if trial, err = trial.Sign(r.serverName, r.signingKey); err != nil { + return "", nil, err + } + } + if err := auth.Check(ctx, trial, checkState(authEvents, st), r.verifier()); err != nil { + return "", nil, err + } + + return room.Version, mustMarshal(tmpl), nil +} + +// SendJoinResult is what PUT /send_join answers with. +type SendJoinResult struct { + State []json.RawMessage + AuthChain []json.RawMessage + + // Event is the join countersigned by this server, for a restricted + // join it authorised; nil otherwise. + Event json.RawMessage + + ServersInRoom []string +} + +// SendJoin takes a signed join from another server's user into the room +// and returns the room as of the join. +func (r *API) SendJoin(ctx context.Context, origin, roomID, eventID string, raw json.RawMessage) (SendJoinResult, error) { + e, room, rules, err := r.receiveMembership(ctx, origin, roomID, eventID, raw, event.MembershipJoin) + if err != nil { + return SendJoinResult{}, err + } + + var countersigned json.RawMessage + var content struct { + AuthorisedVia string `json:"join_authorised_via_users_server"` + } + _ = json.Unmarshal(e.Content(), &content) + if content.AuthorisedVia != "" && r.isLocal(content.AuthorisedVia) { + if e, err = e.Sign(r.serverName, r.signingKey); err != nil { + return SendJoinResult{}, err + } + countersigned = e.JSON() + } + + if err := r.receive(ctx, incoming{origin: origin, room: room, rules: rules, event: e, relay: true}); err != nil { + return SendJoinResult{}, err + } + + stored, err := r.store.Event(ctx, roomID, e.EventID()) + if err != nil { + return SendJoinResult{}, err + } + if stored.Outlier { + return SendJoinResult{}, fmt.Errorf("%w: the join was not accepted by the room's current state", ErrForbidden) + } + stateIDs, err := r.stateIDsBefore(ctx, roomID, e.EventID()) + if err != nil { + return SendJoinResult{}, err + } + state, err := r.rawEvents(ctx, roomID, stateIDs) + if err != nil { + return SendJoinResult{}, err + } + chainIDs, err := r.authChainIDs(ctx, roomID, rules, append(slices.Clone(stateIDs), e.AuthEventIDs()...)) + if err != nil { + return SendJoinResult{}, err + } + chain, err := r.rawEvents(ctx, roomID, chainIDs) + if err != nil { + return SendJoinResult{}, err + } + servers, err := r.JoinedServers(ctx, roomID) + if err != nil { + return SendJoinResult{}, err + } + return SendJoinResult{State: state, AuthChain: chain, Event: countersigned, ServersInRoom: servers}, nil +} + +// SendLeave takes a signed leave from another server's user into the +// room. +func (r *API) SendLeave(ctx context.Context, origin, roomID, eventID string, raw json.RawMessage) error { + e, room, rules, err := r.receiveMembership(ctx, origin, roomID, eventID, raw, event.MembershipLeave) + if err != nil { + return err + } + return r.receive(ctx, incoming{origin: origin, room: room, rules: rules, event: e, relay: true}) +} + +// receiveMembership checks a send_join or send_leave body: the event the +// path names, a membership event of the right kind, by a user of the +// server sending it, about themselves. +func (r *API) receiveMembership(ctx context.Context, origin, roomID, eventID string, raw json.RawMessage, membership string) (*event.Event, storage.Room, roomversion.Rules, error) { + fail := func(err error) (*event.Event, storage.Room, roomversion.Rules, error) { + return nil, storage.Room{}, roomversion.Rules{}, err + } + if err := r.requireFederation("answering another server's membership request"); err != nil { + return fail(err) + } + room, rules, err := r.roomAndRules(ctx, roomID) + if err != nil { + return fail(err) + } + if in, err := r.participating(ctx, roomID); err != nil { + return fail(err) + } else if !in { + return fail(fmt.Errorf("%w: this server is not in %s", ErrNotInRoom, roomID)) + } + if err := r.requireAllowed(ctx, roomID, origin); err != nil { + return fail(err) + } + e, err := parseRemote(raw, rules) + if err != nil { + return fail(err) + } + stateKey, _ := e.StateKey() + got, _ := e.Membership() + sender, _ := e.SenderServer() + switch { + case e.EventID() != eventID: + return fail(fmt.Errorf("%w: the event is %s, not %s", ErrBadEvent, e.EventID(), eventID)) + case e.RoomID() != roomID: + return fail(fmt.Errorf("%w: the event is in %s, not %s", ErrBadEvent, e.RoomID(), roomID)) + case e.Type() != event.TypeMember || got != membership: + return fail(fmt.Errorf("%w: not a %s event", ErrBadEvent, membership)) + case stateKey != e.Sender(): + return fail(fmt.Errorf("%w: a %s must be the sender's own", ErrBadEvent, membership)) + case sender != origin: + return fail(fmt.Errorf("%w: %s cannot send a %s for %s", ErrForbidden, origin, membership, e.Sender())) + } + return e, room, rules, nil +} + +// ReceiveInvite answers PUT /invite: another server asks this one to +// countersign an invite to one of its users. +func (r *API) ReceiveInvite(ctx context.Context, origin, roomVersion string, raw json.RawMessage, inviteRoomState []json.RawMessage) (json.RawMessage, error) { + if err := r.requireFederation("receiving an invite from another server"); err != nil { + return nil, err + } + rules, err := roomversion.Get(roomversion.Version(roomVersion)) + if err != nil { + return nil, fmt.Errorf("%w: version %q: %v", ErrIncompatibleRoomVersion, roomVersion, err) + } + e, err := parseRemote(raw, rules) + if err != nil { + return nil, err + } + target, _ := e.StateKey() + membership, _ := e.Membership() + sender, _ := e.SenderServer() + switch { + case e.Type() != event.TypeMember || membership != event.MembershipInvite: + return nil, fmt.Errorf("%w: not an invite", ErrBadEvent) + case sender != origin: + return nil, fmt.Errorf("%w: %s cannot send an invite from %s", ErrForbidden, origin, e.Sender()) + case !r.isLocal(target): + return nil, fmt.Errorf("%w: %s is not a user of this server", ErrForbidden, target) + } + if localpart, _ := r.localpartOf(target); r.profiles != nil { + if _, err := r.profiles.Account(ctx, localpart); errors.Is(err, storage.ErrNoSuchUser) { + return nil, fmt.Errorf("%w: %s does not exist", ErrNoSuchEvent, target) + } else if err != nil { + return nil, err + } + } + + verified, err := r.verifyIncoming(ctx, e) + if err != nil { + return nil, err + } + signed, err := verified.Sign(r.serverName, r.signingKey) + if err != nil { + return nil, err + } + roomID := signed.RoomID() + + // A room this server is in hears about the invite like any other + // event, and judges it the same way. One it is not in gets the invite + // as an out-of-band membership with the state the inviter sent along. + in, err := r.participating(ctx, roomID) + if err != nil { + return nil, err + } + if in { + if err := r.requireAllowed(ctx, roomID, origin); err != nil { + return nil, err + } + room, rules, err := r.roomAndRules(ctx, roomID) + if err != nil { + return nil, err + } + if err := r.receive(ctx, incoming{origin: origin, room: room, rules: rules, event: signed, fetchMissing: true}); err != nil { + return nil, err + } + return signed.JSON(), nil + } + + if err := r.store.CreateRoom(ctx, storage.NewRoom{ + ID: roomID, Version: roomVersion, Creator: signed.Sender(), CreatedAtMS: r.nowMS(), + }); err != nil && !errors.Is(err, storage.ErrRoomExists) { + return nil, err + } + room, err := r.store.Room(ctx, roomID) + if err != nil { + return nil, err + } + var stripped []json.RawMessage + for _, s := range inviteRoomState { + var probe map[string]json.RawMessage + if json.Unmarshal(s, &probe) == nil && probe != nil { + stripped = append(stripped, s) + } + } + if err := r.persistOutOfBand(ctx, room, signed, mustMarshal(orEmptyRaw(stripped)), origin); err != nil { + return nil, err + } + r.log.Info("received an invite over federation", + "room_id", roomID, "user_id", target, "sender", signed.Sender()) + return signed.JSON(), nil +} + +func orEmptyRaw(s []json.RawMessage) []json.RawMessage { + if s == nil { + return []json.RawMessage{} + } + return s +} diff --git a/server/internal/roomserver/federated_query.go b/server/internal/roomserver/federated_query.go new file mode 100644 index 0000000..200b9fa --- /dev/null +++ b/server/internal/roomserver/federated_query.go @@ -0,0 +1,373 @@ +package roomserver + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "sort" + + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// The read side of the federation API: other servers asking about a room +// this one is in. +// +// Every read is answered only for a server that is in the room itself, or +// for a room anybody may read. A server outside the room asking for its +// events is asking for history its users were never shown, and the answer +// is the same 403 whether or not the event exists. + +// Bounds on the federation reads. +const ( + maxBackfillLimit = 100 + maxMissingEventsLimit = 20 +) + +// requireServerInRoom refuses a server that may not read a room. +func (r *API) requireServerInRoom(ctx context.Context, roomID, origin string) (storage.Room, roomversion.Rules, error) { + if err := r.requireFederation("answering another server"); err != nil { + return storage.Room{}, roomversion.Rules{}, err + } + room, rules, err := r.roomAndRules(ctx, roomID) + if err != nil { + return storage.Room{}, roomversion.Rules{}, err + } + if err := r.requireAllowed(ctx, roomID, origin); err != nil { + return storage.Room{}, roomversion.Rules{}, err + } + in, err := r.ServerInRoom(ctx, roomID, origin) + if err != nil { + return storage.Room{}, roomversion.Rules{}, err + } + if in { + return room, rules, nil + } + // A world-readable room may be read by anybody, servers included. + if stored, err := r.store.CurrentStateEvent(ctx, roomID, event.TypeHistoryVisibility, ""); err == nil { + var content struct { + HistoryVisibility string `json:"history_visibility"` + } + if json.Unmarshal(contentOf(stored), &content) == nil && content.HistoryVisibility == HistoryVisibilityWorldReadable { + return room, rules, nil + } + } + return storage.Room{}, roomversion.Rules{}, fmt.Errorf("%w: %s is not in %s", ErrForbidden, origin, roomID) +} + +// contentOf reads the content of a stored event without parsing it as a +// PDU, for the one property a visibility check needs. +func contentOf(stored storage.StoredEvent) json.RawMessage { + var probe struct { + Content json.RawMessage `json:"content"` + } + _ = json.Unmarshal(stored.JSON, &probe) + return probe.Content +} + +// EventForServer answers GET /event: one event, as JSON. +func (r *API) EventForServer(ctx context.Context, origin, eventID string) (json.RawMessage, error) { + stored, err := r.store.EventByID(ctx, eventID) + if err != nil { + return nil, err + } + if _, _, err := r.requireServerInRoom(ctx, stored.RoomID, origin); err != nil { + return nil, err + } + return stored.JSON, nil +} + +// StateIDsForServer answers GET /state_ids: the state before an event and +// its auth chain, as event IDs. +func (r *API) StateIDsForServer(ctx context.Context, origin, roomID, eventID string) ([]string, []string, error) { + _, rules, err := r.requireServerInRoom(ctx, roomID, origin) + if err != nil { + return nil, nil, err + } + stateIDs, err := r.stateIDsBefore(ctx, roomID, eventID) + if err != nil { + return nil, nil, err + } + chain, err := r.authChainIDs(ctx, roomID, rules, stateIDs) + if err != nil { + return nil, nil, err + } + return stateIDs, chain, nil +} + +// StateForServer answers GET /state: StateIDsForServer with the events. +func (r *API) StateForServer(ctx context.Context, origin, roomID, eventID string) ([]json.RawMessage, []json.RawMessage, error) { + stateIDs, chainIDs, err := r.StateIDsForServer(ctx, origin, roomID, eventID) + if err != nil { + return nil, nil, err + } + state, err := r.rawEvents(ctx, roomID, stateIDs) + if err != nil { + return nil, nil, err + } + chain, err := r.rawEvents(ctx, roomID, chainIDs) + if err != nil { + return nil, nil, err + } + return state, chain, nil +} + +// EventAuthForServer answers GET /event_auth: an event's auth chain. +func (r *API) EventAuthForServer(ctx context.Context, origin, roomID, eventID string) ([]json.RawMessage, error) { + _, rules, err := r.requireServerInRoom(ctx, roomID, origin) + if err != nil { + return nil, err + } + stored, err := r.store.Event(ctx, roomID, eventID) + if err != nil { + return nil, err + } + e, err := parseEvent(stored, rules) + if err != nil { + return nil, err + } + ids, err := r.authChainIDs(ctx, roomID, rules, e.AuthEventIDs()) + if err != nil { + return nil, err + } + return r.rawEvents(ctx, roomID, ids) +} + +// stateIDsBefore returns the event IDs of the state before an event — the +// snapshot it was authorised against. +func (r *API) stateIDsBefore(ctx context.Context, roomID, eventID string) ([]string, error) { + stored, err := r.store.Event(ctx, roomID, eventID) + if err != nil { + return nil, err + } + if stored.Outlier { + // An outlier has no snapshot; the honest answer is that this + // server does not know the state at it. + return nil, fmt.Errorf("%w: the state at %s is not known here", ErrNoSuchEvent, eventID) + } + entries, err := r.store.StateBefore(ctx, roomID, eventID) + if err != nil { + return nil, err + } + return stateEntryIDs(entries), nil +} + +// authChainIDs walks auth_events from a set of events to the create event +// and returns every event on the way, the starting events excluded unless +// one of the others cites them. +func (r *API) authChainIDs(ctx context.Context, roomID string, rules roomversion.Rules, from []string) ([]string, error) { + seen := map[string]bool{} + frontier := map[string]bool{} + start := map[string]*event.Event{} + + starting, err := r.eventsByID(ctx, roomID, rules, from) + if err != nil { + return nil, err + } + for _, e := range starting { + start[e.EventID()] = e + for _, id := range e.AuthEventIDs() { + frontier[id] = true + } + } + for _, id := range from { + if _, ok := start[id]; !ok { + // A starting ID that is not an event here is somebody's auth + // event reference rather than an event of the chain's own. + frontier[id] = true + } + } + + for len(frontier) > 0 { + ids := make([]string, 0, len(frontier)) + for id := range frontier { + if !seen[id] { + seen[id] = true + ids = append(ids, id) + } + } + frontier = map[string]bool{} + if len(ids) == 0 { + break + } + events, err := r.eventsByID(ctx, roomID, rules, ids) + if err != nil { + return nil, err + } + for _, e := range events { + for _, id := range e.AuthEventIDs() { + if !seen[id] { + frontier[id] = true + } + } + } + } + + // Only the IDs that are events here: a reference to something this + // server never had is not part of the answer. + all := make([]string, 0, len(seen)) + for id := range seen { + all = append(all, id) + } + present, err := r.store.EventsByID(ctx, roomID, all) + if err != nil { + return nil, err + } + out := make([]string, 0, len(present)) + for _, s := range present { + out = append(out, s.EventID) + } + sort.Strings(out) + return out, nil +} + +// rawEvents reads events as the JSON they are held as. +func (r *API) rawEvents(ctx context.Context, roomID string, ids []string) ([]json.RawMessage, error) { + if len(ids) == 0 { + return []json.RawMessage{}, nil + } + stored, err := r.store.EventsByID(ctx, roomID, ids) + if err != nil { + return nil, err + } + out := make([]json.RawMessage, 0, len(stored)) + for _, s := range stored { + out = append(out, s.JSON) + } + return out, nil +} + +// BackfillForServer answers GET /backfill: up to limit events reachable +// backwards through prev_events from the given ones, them included, +// newest first. +func (r *API) BackfillForServer(ctx context.Context, origin, roomID string, from []string, limit int) ([]json.RawMessage, error) { + _, rules, err := r.requireServerInRoom(ctx, roomID, origin) + if err != nil { + return nil, err + } + if limit <= 0 || limit > maxBackfillLimit { + limit = maxBackfillLimit + } + events, err := r.walkBack(ctx, roomID, rules, from, nil, 0, limit) + if err != nil { + return nil, err + } + sort.SliceStable(events, func(i, j int) bool { return events[i].Depth() > events[j].Depth() }) + return jsonOf(events), nil +} + +// MissingEventsForServer answers POST /get_missing_events: the events +// between earliest and latest, oldest first. +func (r *API) MissingEventsForServer(ctx context.Context, origin, roomID string, earliest, latest []string, limit int, minDepth int64) ([]json.RawMessage, error) { + _, rules, err := r.requireServerInRoom(ctx, roomID, origin) + if err != nil { + return nil, err + } + if limit <= 0 || limit > maxMissingEventsLimit { + limit = maxMissingEventsLimit + } + starting, err := r.eventsByID(ctx, roomID, rules, latest) + if err != nil { + return nil, err + } + var from []string + for _, e := range starting { + from = append(from, e.PrevEventIDs()...) + } + events, err := r.walkBack(ctx, roomID, rules, from, earliest, minDepth, limit) + if err != nil { + return nil, err + } + eventsByDepth(events) + return jsonOf(events), nil +} + +// walkBack walks prev_events breadth first from a set of events, stopping +// at the ones in stop, below minDepth, or after limit events. +// +// Outliers are skipped. Their prev_events were never fetched, and handing +// one out would invite the asker to come back for history this server +// does not have. +func (r *API) walkBack(ctx context.Context, roomID string, rules roomversion.Rules, from, stop []string, minDepth int64, limit int) ([]*event.Event, error) { + seen := map[string]bool{} + for _, id := range stop { + seen[id] = true + } + var out []*event.Event + frontier := slices.Clone(from) + for len(frontier) > 0 && len(out) < limit { + var ids []string + for _, id := range frontier { + if !seen[id] { + seen[id] = true + ids = append(ids, id) + } + } + frontier = nil + if len(ids) == 0 { + break + } + stored, err := r.store.EventsByID(ctx, roomID, ids) + if err != nil { + return nil, err + } + var level []*event.Event + for _, s := range stored { + if s.Outlier { + continue + } + e, err := parseEvent(s, rules) + if err != nil { + return nil, err + } + if e.Depth() < minDepth { + continue + } + level = append(level, e) + } + // Newest first within a level, so that the limit keeps the events + // nearest to where the walk started. + sort.SliceStable(level, func(i, j int) bool { return level[i].Depth() > level[j].Depth() }) + for _, e := range level { + if len(out) >= limit { + break + } + out = append(out, e) + frontier = append(frontier, e.PrevEventIDs()...) + } + } + return out, nil +} + +func jsonOf(events []*event.Event) []json.RawMessage { + out := make([]json.RawMessage, 0, len(events)) + for _, e := range events { + out = append(out, e.JSON()) + } + return out +} + +// QueryAliasForServer answers GET /query/directory for an alias of this +// server. +func (r *API) QueryAliasForServer(ctx context.Context, alias string) (ResolvedAlias, error) { + if err := r.requireFederation("answering another server"); err != nil { + return ResolvedAlias{}, err + } + if !r.isLocalAlias(alias) { + return ResolvedAlias{}, fmt.Errorf("%w: %s is not an alias of this server", ErrNoSuchAlias, alias) + } + stored, err := r.store.Alias(ctx, alias) + if err != nil { + return ResolvedAlias{}, err + } + servers, err := r.JoinedServers(ctx, stored.RoomID) + if err != nil { + return ResolvedAlias{}, err + } + if !slices.Contains(servers, r.serverName) { + servers = append([]string{r.serverName}, servers...) + } + return ResolvedAlias{RoomID: stored.RoomID, Servers: servers}, nil +} diff --git a/server/internal/roomserver/federated_test.go b/server/internal/roomserver/federated_test.go new file mode 100644 index 0000000..df39753 --- /dev/null +++ b/server/internal/roomserver/federated_test.go @@ -0,0 +1,1054 @@ +package roomserver_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "path/filepath" + "slices" + "sync" + "testing" + "time" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" + "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// The federation tests run several roomservers in one process, each on its +// own database, connected by a transport that calls the other server's +// federation methods directly. Nothing about HTTP or X-Matrix is under +// test here — the federation package has its own tests for that — only +// whether two roomservers that can reach each other end up agreeing about +// a room. + +// mesh is a set of servers that can reach each other. +type mesh struct { + t *testing.T + servers map[string]*peer + + // disabled turns federation off everywhere, for the switch tests. + mu sync.Mutex + disabled bool +} + +// peer is one server of the mesh. +type peer struct { + name string + api *roomserver.API + store storage.RoomStore + key *signing.KeyPair + mesh *mesh + + mu sync.Mutex + // outbox is every event this server persisted that it would send: + // its own, and the ones it relays. + outbox []roomserver.Notification +} + +func newMesh(t *testing.T, names ...string) *mesh { + t.Helper() + m := &mesh{t: t, servers: map[string]*peer{}} + for _, name := range names { + key, err := signing.Generate("k_" + name[:1]) + if err != nil { + t.Fatal(err) + } + m.servers[name] = &peer{name: name, key: key, mesh: m} + } + for _, name := range names { + m.servers[name].start(t) + } + return m +} + +func (p *peer) start(t *testing.T) { + t.Helper() + path := filepath.ToSlash(filepath.Join(t.TempDir(), p.name+".db")) + db, err := storage.Open(t.Context(), storage.Config{ + Driver: storage.DriverSQLite, + ConnectionString: "file:" + path + "?_pragma=busy_timeout(5000)", + }, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatalf("open storage: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if err := db.Migrate(t.Context()); err != nil { + t.Fatalf("migrate: %v", err) + } + + api, err := roomserver.New(roomserver.Options{ + ServerName: p.name, + Store: db.Rooms(), + SigningKey: p.key, + Profiles: anyProfile{}, + FederationEnabled: func() bool { return !p.mesh.isDisabled() }, + Notifier: p, + Remote: transport{from: p}, + Keys: keys{m: p.mesh}, + Logger: slog.New(slog.DiscardHandler), + }) + if err != nil { + t.Fatalf("roomserver.New: %v", err) + } + p.api, p.store = api, db.Rooms() +} + +func (m *mesh) isDisabled() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.disabled +} + +// anyProfile gives every local user a display name. +type anyProfile struct{} + +func (anyProfile) Account(_ context.Context, localpart string) (storage.Account, error) { + return storage.Account{Localpart: localpart, DisplayName: "User " + localpart}, nil +} + +// EventPersisted implements roomserver.Notifier: it fills the outbox the +// way the real outbound queue picks its events. +func (p *peer) EventPersisted(_ context.Context, n roomserver.Notification) { + if n.Origin != "" && !n.Relay { + return + } + p.mu.Lock() + defer p.mu.Unlock() + p.outbox = append(p.outbox, n) +} + +// takeOutbox empties the outbox. +func (p *peer) takeOutbox() []roomserver.Notification { + p.mu.Lock() + defer p.mu.Unlock() + out := p.outbox + p.outbox = nil + return out +} + +// flush delivers every outbox to the servers in each event's room, the +// way the queue would, until nothing is left to send. It returns the +// errors the receivers reported, keyed by event ID. +func (m *mesh) flush() map[string]error { + m.t.Helper() + failures := map[string]error{} + for range 20 { + sent := false + for _, name := range slices.Sorted(mapsKeys(m.servers)) { + p := m.servers[name] + for _, n := range p.takeOutbox() { + sent = true + m.deliver(p, n, failures) + } + } + if !sent { + return failures + } + } + m.t.Fatal("flush did not settle") + return nil +} + +// deliver sends one event from a server to the others in its room. +func (m *mesh) deliver(from *peer, n roomserver.Notification, failures map[string]error) { + m.t.Helper() + servers, err := from.api.JoinedServers(m.t.Context(), n.RoomID) + if err != nil { + m.t.Fatalf("JoinedServers: %v", err) + } + // A membership event also goes to the server of the user it is + // about, so that a kicked or banned server hears of it. + if key, ok := n.Event.StateKey(); ok && n.Event.Type() == event.TypeMember { + if server, err := id.ServerNameOf(key); err == nil && !slices.Contains(servers, server) { + servers = append(servers, server) + } + } + for _, server := range servers { + if server == from.name || server == n.Origin { + continue + } + to, ok := m.servers[server] + if !ok { + continue + } + if _, err := to.api.ReceivePDU(m.t.Context(), from.name, n.Event.JSON()); err != nil { + failures[n.Event.EventID()] = err + } + } +} + +// deliverOnly sends one specific event, bypassing the outbox: the tests +// that leave gaps use it. +func (m *mesh) deliverOnly(from, to string, e *event.Event) error { + _, err := m.servers[to].api.ReceivePDU(m.t.Context(), from, e.JSON()) + return err +} + +func mapsKeys[V any](m map[string]V) func(func(string) bool) { + return func(yield func(string) bool) { + for k := range m { + if !yield(k) { + return + } + } + } +} + +// keys verifies signatures with the mesh's keys, standing in for the key +// ring. +type keys struct{ m *mesh } + +func (k keys) VerifyEventSignature(_ context.Context, serverName string, e *event.Event) error { + p, ok := k.m.servers[serverName] + if !ok { + return fmt.Errorf("no key for %s", serverName) + } + return e.VerifySignature(serverName, p.key.KeyID(), p.key.Public) +} + +// transport is RemoteServers for one peer: every call goes straight to +// the destination's federation-side method, with this peer as the origin. +type transport struct{ from *peer } + +func (t transport) to(destination string) (*roomserver.API, error) { + if t.from.mesh.isDisabled() { + return nil, federation.ErrDisabled + } + p, ok := t.from.mesh.servers[destination] + if !ok { + return nil, fmt.Errorf("%s is unreachable", destination) + } + return p.api, nil +} + +// httpError turns a roomserver error into what the HTTP layer would have +// sent back, so that the caller's error mapping is exercised. +func httpError(err error) error { + switch { + case err == nil: + return nil + case errors.Is(err, roomserver.ErrIncompatibleRoomVersion): + return &federation.HTTPError{Status: http.StatusBadRequest, Code: "M_INCOMPATIBLE_ROOM_VERSION", Message: err.Error()} + case errors.Is(err, roomserver.ErrForbidden), errors.Is(err, roomserver.ErrRejected), errors.Is(err, roomserver.ErrNotInRoom): + return &federation.HTTPError{Status: http.StatusForbidden, Code: "M_FORBIDDEN", Message: err.Error()} + case errors.Is(err, roomserver.ErrNoSuchRoom), errors.Is(err, roomserver.ErrNoSuchEvent), errors.Is(err, roomserver.ErrNoSuchAlias): + return &federation.HTTPError{Status: http.StatusNotFound, Code: "M_NOT_FOUND", Message: err.Error()} + } + return &federation.HTTPError{Status: http.StatusInternalServerError, Code: "M_UNKNOWN", Message: err.Error()} +} + +func (t transport) MakeJoin(ctx context.Context, dest, roomID, userID string, versions []string) (federation.MakeJoinResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.MakeJoinResponse{}, err + } + version, tmpl, err := api.MakeJoin(ctx, t.from.name, roomID, userID, versions) + return federation.MakeJoinResponse{RoomVersion: version, Event: tmpl}, httpError(err) +} + +func (t transport) SendJoin(ctx context.Context, dest, roomID, eventID string, raw json.RawMessage) (federation.SendJoinResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.SendJoinResponse{}, err + } + res, err := api.SendJoin(ctx, t.from.name, roomID, eventID, raw) + return federation.SendJoinResponse{Origin: dest, State: res.State, AuthChain: res.AuthChain, Event: res.Event}, httpError(err) +} + +func (t transport) MakeLeave(ctx context.Context, dest, roomID, userID string) (federation.MakeJoinResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.MakeJoinResponse{}, err + } + version, tmpl, err := api.MakeLeave(ctx, t.from.name, roomID, userID) + return federation.MakeJoinResponse{RoomVersion: version, Event: tmpl}, httpError(err) +} + +func (t transport) SendLeave(ctx context.Context, dest, roomID, eventID string, raw json.RawMessage) error { + api, err := t.to(dest) + if err != nil { + return err + } + return httpError(api.SendLeave(ctx, t.from.name, roomID, eventID, raw)) +} + +func (t transport) Invite(ctx context.Context, dest, _, _ string, req federation.InviteRequest) (federation.InviteResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.InviteResponse{}, err + } + signed, err := api.ReceiveInvite(ctx, t.from.name, req.RoomVersion, req.Event, req.InviteRoomState) + return federation.InviteResponse{Event: signed}, httpError(err) +} + +func (t transport) Event(ctx context.Context, dest, eventID string) (federation.Transaction, error) { + api, err := t.to(dest) + if err != nil { + return federation.Transaction{}, err + } + raw, err := api.EventForServer(ctx, t.from.name, eventID) + if err != nil { + return federation.Transaction{}, httpError(err) + } + return federation.Transaction{Origin: dest, PDUs: []json.RawMessage{raw}}, nil +} + +func (t transport) EventAuth(ctx context.Context, dest, roomID, eventID string) (federation.EventAuthResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.EventAuthResponse{}, err + } + chain, err := api.EventAuthForServer(ctx, t.from.name, roomID, eventID) + return federation.EventAuthResponse{AuthChain: chain}, httpError(err) +} + +func (t transport) StateIDs(ctx context.Context, dest, roomID, eventID string) (federation.StateIDsResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.StateIDsResponse{}, err + } + state, chain, err := api.StateIDsForServer(ctx, t.from.name, roomID, eventID) + return federation.StateIDsResponse{PDUIDs: state, AuthChainIDs: chain}, httpError(err) +} + +func (t transport) State(ctx context.Context, dest, roomID, eventID string) (federation.StateResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.StateResponse{}, err + } + state, chain, err := api.StateForServer(ctx, t.from.name, roomID, eventID) + return federation.StateResponse{PDUs: state, AuthChain: chain}, httpError(err) +} + +func (t transport) MissingEvents(ctx context.Context, dest, roomID string, req federation.MissingEventsRequest) (federation.MissingEventsResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.MissingEventsResponse{}, err + } + events, err := api.MissingEventsForServer(ctx, t.from.name, roomID, req.EarliestEvents, req.LatestEvents, req.Limit, req.MinDepth) + return federation.MissingEventsResponse{Events: events}, httpError(err) +} + +func (t transport) QueryDirectory(ctx context.Context, dest, alias string) (federation.DirectoryResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.DirectoryResponse{}, err + } + resolved, err := api.QueryAliasForServer(ctx, alias) + return federation.DirectoryResponse{RoomID: resolved.RoomID, Servers: resolved.Servers}, httpError(err) +} + +func (t transport) PublicRooms(ctx context.Context, dest string, limit int, since, search string) (json.RawMessage, error) { + api, err := t.to(dest) + if err != nil { + return nil, err + } + resp, err := api.PublicRooms(ctx, roomserver.PublicRoomsRequest{Limit: limit, Since: since, Search: search}) + if err != nil { + return nil, httpError(err) + } + return json.Marshal(resp) +} + +// --------------------------------------------------------------------------- + +const ( + serverA = "a.example" + serverB = "b.example" + serverC = "c.example" + + userA = "@alice:" + serverA + userA2 = "@anne:" + serverA + userB = "@bob:" + serverB + userC = "@cath:" + serverC +) + +// send sends a message and returns its event ID. +func (p *peer) send(t *testing.T, roomID, sender, body string) string { + t.Helper() + eventID, err := p.api.SendEvent(t.Context(), roomserver.SendEventRequest{ + RoomID: roomID, Sender: sender, Type: "m.room.message", + Content: json.RawMessage(`{"msgtype":"m.text","body":"` + body + `"}`), + }) + if err != nil { + t.Fatalf("%s: SendEvent: %v", p.name, err) + } + return eventID +} + +// timelineIDs returns the event IDs of a room's timeline on a server, +// oldest first. +func (p *peer) timelineIDs(t *testing.T, roomID string) []string { + t.Helper() + events, err := p.store.Timeline(t.Context(), storage.TimelineRequest{ + RoomID: roomID, Limit: 1000, Direction: storage.DirectionForward, + }) + if err != nil { + t.Fatalf("Timeline: %v", err) + } + ids := make([]string, 0, len(events)) + for _, e := range events { + ids = append(ids, e.EventID) + } + return ids +} + +// membership reads a user's membership on a server. +func (p *peer) membership(t *testing.T, roomID, userID string) string { + t.Helper() + m, err := p.api.Membership(t.Context(), roomID, userID) + if err != nil && !errors.Is(err, roomserver.ErrNoSuchRoom) { + t.Fatalf("Membership: %v", err) + } + return m +} + +// currentState returns a server's current state of a room as +// type/state_key -> event ID. +func (p *peer) currentState(t *testing.T, roomID string) map[string]string { + t.Helper() + entries, err := p.store.CurrentState(t.Context(), roomID) + if err != nil { + t.Fatalf("CurrentState: %v", err) + } + out := map[string]string{} + for _, e := range entries { + out[e.Type+"|"+e.StateKey] = e.EventID + } + return out +} + +// publicRoom creates a public room on a server. +func (p *peer) publicRoom(t *testing.T, creator string) string { + t.Helper() + resp, err := p.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{ + Creator: creator, Preset: roomserver.PresetPublicChat, Name: "Federated", + }) + if err != nil { + t.Fatalf("CreateRoom: %v", err) + } + return resp.RoomID +} + +func TestFederatedJoinAndConversation(t *testing.T) { + for _, version := range []string{"10", "11", "12"} { + t.Run("v"+version, func(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{ + Creator: userA, Preset: roomserver.PresetPublicChat, Name: "Federated", Version: version, + }) + if err != nil { + t.Fatalf("CreateRoom: %v", err) + } + roomID := resp.RoomID + a.takeOutbox() // nobody else is in the room yet + + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{ + RoomID: roomID, User: userB, ServerNames: []string{serverA}, + }); err != nil { + t.Fatalf("remote join: %v", err) + } + if got := b.membership(t, roomID, userB); got != event.MembershipJoin { + t.Fatalf("B sees its user as %q", got) + } + if got := a.membership(t, roomID, userB); got != event.MembershipJoin { + t.Fatalf("A sees B's user as %q", got) + } + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + + // Both servers hold the same room. + if sa, sb := a.currentState(t, roomID), b.currentState(t, roomID); !mapsEqual(sa, sb) { + t.Fatalf("current state differs:\nA: %v\nB: %v", sa, sb) + } + summary, err := b.store.RoomSummary(t.Context(), roomID) + if err != nil { + t.Fatalf("RoomSummary: %v", err) + } + if summary.Name != "Federated" || summary.JoinedMembers != 2 { + t.Errorf("B's listing of the room = %+v", summary.RoomProfile) + } + + // A conversation in both directions. + m1 := a.send(t, roomID, userA, "hello from A") + m2 := b.send(t, roomID, userB, "hello from B") + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + for _, p := range []*peer{a, b} { + ids := p.timelineIDs(t, roomID) + if !slices.Contains(ids, m1) || !slices.Contains(ids, m2) { + t.Errorf("%s's timeline lacks a message: %v", p.name, ids) + } + } + + // The two messages were sent concurrently, so each server now + // has two forward extremities; the next event joins them. + m3 := a.send(t, roomID, userA, "merge") + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + for _, p := range []*peer{a, b} { + exts, err := p.store.Extremities(t.Context(), roomID) + if err != nil { + t.Fatal(err) + } + if len(exts) != 1 || exts[0].EventID != m3 { + t.Errorf("%s's extremities after the merge = %v", p.name, eventIDsOf(exts)) + } + } + if sa, sb := a.currentState(t, roomID), b.currentState(t, roomID); !mapsEqual(sa, sb) { + t.Errorf("current state differs after the conversation:\nA: %v\nB: %v", sa, sb) + } + }) + } +} + +func TestRemoteJoinFailsOverToTheNextServer(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{ + RoomID: roomID, User: userB, ServerNames: []string{"gone.example", serverA}, + }); err != nil { + t.Fatalf("join with an unreachable first server: %v", err) + } + if got := b.membership(t, roomID, userB); got != event.MembershipJoin { + t.Errorf("membership = %q", got) + } +} + +func TestRemoteJoinToAnInviteOnlyRoomIsRefused(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{Creator: userA, Preset: roomserver.PresetPrivateChat}) + if err != nil { + t.Fatal(err) + } + + _, err = b.api.JoinRoom(t.Context(), roomserver.JoinRequest{ + RoomID: resp.RoomID, User: userB, ServerNames: []string{serverA}, + }) + if !errors.Is(err, roomserver.ErrForbidden) { + t.Fatalf("join to an invite-only room = %v, want ErrForbidden", err) + } + if _, err := b.store.Room(t.Context(), resp.RoomID); !errors.Is(err, storage.ErrNoSuchRoom) { + t.Errorf("a refused join left the room behind on B: %v", err) + } +} + +func TestRemoteInviteJoinAndReject(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{ + Creator: userA, Preset: roomserver.PresetPrivateChat, Name: "Secret", + }) + if err != nil { + t.Fatal(err) + } + roomID := resp.RoomID + + if err := a.api.InviteUser(t.Context(), roomserver.InviteRequest{RoomID: roomID, Sender: userA, Target: userB}); err != nil { + t.Fatalf("remote invite: %v", err) + } + if got := b.membership(t, roomID, userB); got != event.MembershipInvite { + t.Fatalf("B sees its user as %q after the invite", got) + } + // The invite on A carries both servers' signatures. + stored, err := a.store.CurrentStateEvent(t.Context(), roomID, event.TypeMember, userB) + if err != nil { + t.Fatal(err) + } + var sigs struct { + Signatures map[string]any `json:"signatures"` + } + _ = json.Unmarshal(stored.JSON, &sigs) + if _, ok := sigs.Signatures[serverB]; !ok { + t.Error("the invite A holds is not countersigned by B") + } + + // B shows the invite with the state A sent along. + stripped, err := b.api.StrippedState(t.Context(), roomID, []string{userB}) + if err != nil { + t.Fatalf("StrippedState: %v", err) + } + var sawName, sawInvite bool + for _, s := range stripped { + if s.Type == "m.room.name" { + sawName = true + } + if s.Type == event.TypeMember && s.StateKey == userB { + sawInvite = true + } + } + if !sawName || !sawInvite { + t.Errorf("B's invite state = %+v", stripped) + } + + // Accepting goes through A, which is the inviter. + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB}); err != nil { + t.Fatalf("accepting the invite: %v", err) + } + if got := a.membership(t, roomID, userB); got != event.MembershipJoin { + t.Errorf("A sees B's user as %q after the join", got) + } + if got, _ := b.store.InviteState(t.Context(), roomID, userB); got != nil { + t.Error("the invite state outlived the invite") + } + + // B is in the room now, so a second invite to one of its users is + // taken in as an ordinary event of the room rather than out of band. + m.flush() + if err := a.api.InviteUser(t.Context(), roomserver.InviteRequest{RoomID: roomID, Sender: userA, Target: "@zed:" + serverB}); err != nil { + t.Fatalf("second invite: %v", err) + } + m.flush() + if got := b.membership(t, roomID, "@zed:"+serverB); got != event.MembershipInvite { + t.Fatalf("B sees zed as %q", got) + } +} + +func TestRejectingARemoteInvite(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{Creator: userA, Preset: roomserver.PresetPrivateChat}) + if err != nil { + t.Fatal(err) + } + roomID := resp.RoomID + if err := a.api.InviteUser(t.Context(), roomserver.InviteRequest{RoomID: roomID, Sender: userA, Target: userB}); err != nil { + t.Fatal(err) + } + + if err := b.api.LeaveRoom(t.Context(), roomID, userB, "no thanks"); err != nil { + t.Fatalf("rejecting: %v", err) + } + if got := b.membership(t, roomID, userB); got != event.MembershipLeave { + t.Errorf("B sees its user as %q after rejecting", got) + } + if got := a.membership(t, roomID, userB); got != event.MembershipLeave { + t.Errorf("A sees B's user as %q after the rejection", got) + } +} + +func TestRejectingAnInviteFromAnUnreachableServer(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{Creator: userA, Preset: roomserver.PresetPrivateChat}) + if err != nil { + t.Fatal(err) + } + if err := a.api.InviteUser(t.Context(), roomserver.InviteRequest{RoomID: resp.RoomID, Sender: userA, Target: userB}); err != nil { + t.Fatal(err) + } + delete(m.servers, serverA) + + if err := b.api.LeaveRoom(t.Context(), resp.RoomID, userB, ""); err != nil { + t.Fatalf("rejecting locally: %v", err) + } + if got := b.membership(t, resp.RoomID, userB); got != event.MembershipLeave { + t.Errorf("membership after a local rejection = %q", got) + } +} + +func TestMissingEventsAreFetched(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); err != nil { + t.Fatal(err) + } + m.flush() + + var sent []string + for i := range 4 { + sent = append(sent, a.send(t, roomID, userA, fmt.Sprint("m", i))) + } + outbox := a.takeOutbox() + // Only the last one gets through; B has to ask for the other three. + last := outbox[len(outbox)-1] + if err := m.deliverOnly(serverA, serverB, last.Event); err != nil { + t.Fatalf("delivering the last message: %v", err) + } + ids := b.timelineIDs(t, roomID) + for _, id := range sent { + if !slices.Contains(ids, id) { + t.Errorf("B's timeline lacks %s", id) + } + } + // In order. + var positions []int + for _, id := range sent { + positions = append(positions, slices.Index(ids, id)) + } + if !slices.IsSorted(positions) { + t.Errorf("fetched events are out of order: %v", positions) + } +} + +func TestAWideGapIsClosedWithState(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); err != nil { + t.Fatal(err) + } + m.flush() + + // More than get_missing_events will be asked for, with a state change + // in the part that is not fetched. + for i := range 6 { + a.send(t, roomID, userA, fmt.Sprint("early", i)) + } + if _, err := a.api.SendEvent(t.Context(), roomserver.SendEventRequest{ + RoomID: roomID, Sender: userA, Type: "m.room.topic", StateKey: ptrTo(""), + Content: json.RawMessage(`{"topic":"changed in the gap"}`), + }); err != nil { + t.Fatal(err) + } + for i := range 15 { + a.send(t, roomID, userA, fmt.Sprint("late", i)) + } + outbox := a.takeOutbox() + last := outbox[len(outbox)-1] + if err := m.deliverOnly(serverA, serverB, last.Event); err != nil { + t.Fatalf("delivering across the gap: %v", err) + } + + if !slices.Contains(b.timelineIDs(t, roomID), last.Event.EventID()) { + t.Error("the event across the gap is not in B's timeline") + } + if sa, sb := a.currentState(t, roomID), b.currentState(t, roomID); !mapsEqual(sa, sb) { + t.Errorf("current state differs after the gap:\nA: %v\nB: %v", sa, sb) + } +} + +func TestForgedEventIsRejected(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); err != nil { + t.Fatal(err) + } + m.flush() + + // An event in alice's name, signed by a key that is not A's. + exts, err := b.store.Extremities(t.Context(), roomID) + if err != nil { + t.Fatal(err) + } + impostor, _ := signing.Generate("k_a") + room, _ := b.store.Room(t.Context(), roomID) + rules := mustRules(t, room.Version) + state := b.currentState(t, roomID) + authEvents := []string{state[event.TypePowerLevels+"|"], state[event.TypeMember+"|"+userA]} + if !rules.CreateEventExcludedFromAuthEvents { + authEvents = append(authEvents, state[event.TypeCreate+"|"]) + } + forged, err := (&event.Builder{ + RoomID: roomID, Type: "m.room.message", Sender: userA, + Content: json.RawMessage(`{"body":"not really alice"}`), + PrevEvents: []string{exts[0].EventID}, AuthEvents: authEvents, Depth: exts[0].Depth + 1, + }).Build(fixedNow(), impostor, rules) + if err != nil { + t.Fatal(err) + } + if err := m.deliverOnly(serverA, serverB, forged); !errors.Is(err, roomserver.ErrBadEvent) { + t.Errorf("a forged event = %v, want ErrBadEvent", err) + } + if slices.Contains(b.timelineIDs(t, roomID), forged.EventID()) { + t.Error("the forged event is in the timeline") + } +} + +func TestEventFailingOnlyAgainstCurrentStateIsSoftFailed(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); err != nil { + t.Fatal(err) + } + m.flush() + + // B's user speaks, and the message is held back; meanwhile A bans + // them, and the ban reaches B. + b.send(t, roomID, userB, "last words") + late := b.takeOutbox()[0].Event + if err := a.api.BanUser(t.Context(), roomID, userA, userB, "spam"); err != nil { + t.Fatalf("BanUser: %v", err) + } + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + + // The message was valid when it was sent, and is not now. + if err := m.deliverOnly(serverB, serverA, late); err != nil { + t.Fatalf("a soft-failed event is not an error to its sender: %v", err) + } + stored, err := a.store.Event(t.Context(), roomID, late.EventID()) + if err != nil { + t.Fatalf("the soft-failed event was not kept: %v", err) + } + if !stored.Outlier { + t.Error("the soft-failed event is not an outlier") + } + if slices.Contains(a.timelineIDs(t, roomID), late.EventID()) { + t.Error("the soft-failed event is in A's timeline") + } +} + +func TestFederationSwitchRefusesRemoteWork(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + + m.setDisabled(true) + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); !errors.Is(err, roomserver.ErrFederationDisabled) { + t.Errorf("remote join while disabled = %v, want ErrFederationDisabled", err) + } + if err := a.api.InviteUser(t.Context(), roomserver.InviteRequest{RoomID: roomID, Sender: userA, Target: userB}); !errors.Is(err, roomserver.ErrFederationDisabled) { + t.Errorf("remote invite while disabled = %v, want ErrFederationDisabled", err) + } + if _, _, err := a.api.MakeJoin(t.Context(), serverB, roomID, userB, []string{"11"}); !errors.Is(err, roomserver.ErrFederationDisabled) { + t.Errorf("make_join while disabled = %v, want ErrFederationDisabled", err) + } + + m.setDisabled(false) + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); err != nil { + t.Errorf("remote join after re-enabling: %v", err) + } +} + +func TestThreeServersAgree(t *testing.T) { + m := newMesh(t, serverA, serverB, serverC) + a, b, c := m.servers[serverA], m.servers[serverB], m.servers[serverC] + roomID := a.publicRoom(t, userA) + + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); err != nil { + t.Fatal(err) + } + m.flush() + // C joins through B, which is not the room's creator: B relays the + // join to A. + if _, err := c.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userC, ServerNames: []string{serverB}}); err != nil { + t.Fatalf("C joining through B: %v", err) + } + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + if got := a.membership(t, roomID, userC); got != event.MembershipJoin { + t.Fatalf("A did not hear of C's join through B's relay: %q", got) + } + + a.send(t, roomID, userA2Join(t, a, roomID), "from A") + b.send(t, roomID, userB, "from B") + c.send(t, roomID, userC, "from C") + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + final := a.send(t, roomID, userA, "settle") + m.flush() + + sa, sb, sc := a.currentState(t, roomID), b.currentState(t, roomID), c.currentState(t, roomID) + if !mapsEqual(sa, sb) || !mapsEqual(sa, sc) { + t.Errorf("the three servers disagree:\nA: %v\nB: %v\nC: %v", sa, sb, sc) + } + for _, p := range []*peer{a, b, c} { + if !slices.Contains(p.timelineIDs(t, roomID), final) { + t.Errorf("%s lacks the final message", p.name) + } + } +} + +// userA2Join joins a second local user on A, so that local joins in a +// federated room are covered too, and returns them. +func userA2Join(t *testing.T, a *peer, roomID string) string { + t.Helper() + if _, err := a.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userA2}); err != nil { + t.Fatalf("local join in a federated room: %v", err) + } + return userA2 +} + +func TestRemoteAliasAndDirectory(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{ + Creator: userA, Preset: roomserver.PresetPublicChat, Visibility: roomserver.VisibilityPublic, + RoomAliasName: "lobby", Name: "Lobby", + }) + if err != nil { + t.Fatal(err) + } + + resolved, err := b.api.ResolveAlias(t.Context(), "#lobby:"+serverA) + if err != nil { + t.Fatalf("ResolveAlias: %v", err) + } + if resolved.RoomID != resp.RoomID || !slices.Contains(resolved.Servers, serverA) { + t.Errorf("ResolveAlias = %+v", resolved) + } + if _, err := b.api.ResolveAlias(t.Context(), "#nothing:"+serverA); !errors.Is(err, roomserver.ErrNoSuchAlias) { + t.Errorf("unknown remote alias = %v, want ErrNoSuchAlias", err) + } + + dir, err := b.api.PublicRooms(t.Context(), roomserver.PublicRoomsRequest{Server: serverA}) + if err != nil { + t.Fatalf("PublicRooms: %v", err) + } + if len(dir.Chunk) != 1 || dir.Chunk[0].RoomID != resp.RoomID || dir.Chunk[0].Name != "Lobby" { + t.Errorf("remote directory = %+v", dir) + } +} + +func TestServersOutsideARoomCannotReadIt(t *testing.T) { + m := newMesh(t, serverA, serverB) + a := m.servers[serverA] + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{Creator: userA, Preset: roomserver.PresetPrivateChat}) + if err != nil { + t.Fatal(err) + } + exts, err := a.store.Extremities(t.Context(), resp.RoomID) + if err != nil { + t.Fatal(err) + } + if _, err := a.api.EventForServer(t.Context(), serverB, exts[0].EventID); !errors.Is(err, roomserver.ErrForbidden) { + t.Errorf("/event from outside = %v, want ErrForbidden", err) + } + if _, _, err := a.api.StateIDsForServer(t.Context(), serverB, resp.RoomID, exts[0].EventID); !errors.Is(err, roomserver.ErrForbidden) { + t.Errorf("/state_ids from outside = %v, want ErrForbidden", err) + } + if _, err := a.api.BackfillForServer(t.Context(), serverB, resp.RoomID, []string{exts[0].EventID}, 10); !errors.Is(err, roomserver.ErrForbidden) { + t.Errorf("/backfill from outside = %v, want ErrForbidden", err) + } +} + +func TestBackfillWalksBackwards(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); err != nil { + t.Fatal(err) + } + var sent []string + for i := range 5 { + sent = append(sent, a.send(t, roomID, userA, fmt.Sprint(i))) + } + events, err := a.api.BackfillForServer(t.Context(), serverB, roomID, []string{sent[4]}, 3) + if err != nil { + t.Fatalf("BackfillForServer: %v", err) + } + if len(events) != 3 { + t.Fatalf("backfill returned %d events, want 3", len(events)) + } + var got []string + for _, raw := range events { + var probe struct { + Content struct { + Body string `json:"body"` + } `json:"content"` + } + _ = json.Unmarshal(raw, &probe) + got = append(got, probe.Content.Body) + } + if !slices.Equal(got, []string{"4", "3", "2"}) { + t.Errorf("backfill bodies = %v, want newest first from the starting event", got) + } +} + +func mapsEqual(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} + +func eventIDsOf(events []storage.StoredEvent) []string { + out := make([]string, 0, len(events)) + for _, e := range events { + out = append(out, e.EventID) + } + return out +} + +func ptrTo[T any](v T) *T { return &v } + +func mustRules(t *testing.T, version string) roomversion.Rules { + t.Helper() + rules, err := roomversion.Get(roomversion.Version(version)) + if err != nil { + t.Fatal(err) + } + return rules +} + +func fixedNow() time.Time { return time.Unix(1_700_000_000, 0) } + +func TestServerACLKeepsAServerOut(t *testing.T) { + m := newMesh(t, serverA, serverB, serverC) + a, b, c := m.servers[serverA], m.servers[serverB], m.servers[serverC] + roomID := a.publicRoom(t, userA) + for _, p := range []*peer{b, c} { + user := userB + if p == c { + user = userC + } + if _, err := p.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: user, ServerNames: []string{serverA}}); err != nil { + t.Fatal(err) + } + } + m.flush() + + if _, err := a.api.SendEvent(t.Context(), roomserver.SendEventRequest{ + RoomID: roomID, Sender: userA, Type: roomserver.TypeServerACL, StateKey: ptrTo(""), + Content: json.RawMessage(`{"allow":["*"],"deny":["` + serverC + `"]}`), + }); err != nil { + t.Fatalf("setting the ACL: %v", err) + } + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + + c.send(t, roomID, userC, "from a denied server") + failures := m.flush() + if len(failures) == 0 { + t.Fatal("the denied server's event was accepted") + } + for _, err := range failures { + if !errors.Is(err, roomserver.ErrForbidden) { + t.Errorf("refusal = %v, want ErrForbidden", err) + } + } + if _, _, err := a.api.MakeJoin(t.Context(), serverC, roomID, "@other:"+serverC, []string{"11", "12", "10"}); !errors.Is(err, roomserver.ErrForbidden) { + t.Errorf("make_join from a denied server = %v, want ErrForbidden", err) + } + + // B, which the ACL allows, is unaffected. + b.send(t, roomID, userB, "from an allowed server") + if failures := m.flush(); len(failures) > 0 { + t.Errorf("an allowed server's event was refused: %v", failures) + } +} + +func (m *mesh) setDisabled(disabled bool) { + m.mu.Lock() + defer m.mu.Unlock() + m.disabled = disabled +} diff --git a/server/internal/roomserver/input.go b/server/internal/roomserver/input.go index 64d44a9..4d0956a 100644 --- a/server/internal/roomserver/input.go +++ b/server/internal/roomserver/input.go @@ -41,6 +41,14 @@ type sendRequest struct { // Transaction, when set, records the client transaction ID this event // came from, so that a retried request does not send a second event. Transaction *storage.TransactionRef + + // countersign, when set, is given the event once it has been built and + // authorised and returns the copy to persist. An invite to a user of + // another server goes through it: that server has to sign the invite + // before it can be in the room, and the event persisted is the one with + // both signatures. It runs under the room lock, so the event it signs + // is still a child of the room's current extremities when it lands. + countersign func(context.Context, *event.Event) (*event.Event, error) } // send is the input pipeline for an event this server creates. @@ -125,11 +133,17 @@ func (r *API) send(ctx context.Context, req sendRequest) (*event.Event, error) { // sees this event. For an event created here that state is also the // room's current state, so there is no separate soft-failure check to // make: the two can only differ for an event that arrives from - // somewhere else, which is Phase 5. + // somewhere else, and federated.go checks for that as soft failure. if err := auth.Check(ctx, built, checkState(authEvents, st), r.verifier()); err != nil { return nil, err } + if req.countersign != nil { + if built, err = req.countersign(ctx, built); err != nil { + return nil, err + } + } + if err := r.persist(ctx, persistRequest{ Room: room, Rules: rules, @@ -227,10 +241,9 @@ func nextDepth(prevEvents []*event.Event) int64 { // resolved state has to be written out whole: tuples the resolution dropped // have to go, and an incremental update cannot express that. // -// Only federation forks a room, so in Phase 3 the second path is unreachable -// from outside. It is here rather than in Phase 5 because the alternative is -// a pipeline that authorises against one arbitrary branch of a fork without -// saying so. +// Only federation forks a room. The second path was written in Phase 3, +// before anything could reach it, because the alternative was a pipeline +// that authorised against one arbitrary branch of a fork without saying so. func (r *API) roomState(ctx context.Context, room storage.Room, rules roomversion.Rules) (auth.State, []*event.Event, bool, error) { storedExtremities, err := r.store.Extremities(ctx, room.ID) if err != nil { @@ -312,6 +325,22 @@ type persistRequest struct { Transaction *storage.TransactionRef RedactsEventID string + + // Extremities replaces the room's forward extremities. Nil means the + // event alone, which is right for every event created here: it + // descends from every extremity there was. + Extremities []string + + // CurrentState, when set, is the room's whole current state after the + // event, written as a replacement. An event from another server sets + // it whenever the room's state after it is not simply the state before + // it plus the event: when it forks the room, or when it was built on + // something other than the room's single extremity. + CurrentState auth.State + + // Origin and Relay are passed to the notifier; see Notification. + Origin string + Relay bool } // persist writes the event and everything its arrival changes, then emits. @@ -330,9 +359,12 @@ func (r *API) persist(ctx context.Context, req persistRequest) error { JSON: e.JSON(), }, StateBefore: stateEntries(req.StateBefore), - Extremities: []string{e.EventID()}, + Extremities: req.Extremities, Transaction: req.Transaction, } + if write.Extremities == nil { + write.Extremities = []string{e.EventID()} + } if isState { write.Event.StateKey = &stateKey } @@ -341,11 +373,19 @@ func (r *API) persist(ctx context.Context, req persistRequest) error { // to upsert; after a resolution it is the whole state, which the store // is told to write as a replacement. stateAfter := req.StateBefore - if req.ReplaceWholeState { + stateChanged := isState + switch { + case req.CurrentState != nil: + write.StateDeltaIsWholeState = true + write.StateDelta = stateEntries(req.CurrentState) + stateAfter = req.CurrentState + stateChanged = true + case req.ReplaceWholeState: write.StateDeltaIsWholeState = true write.StateDelta = stateEntries(req.StateBefore) + stateChanged = true } - if isState { + if isState && req.CurrentState == nil { entry := stateEntry(e.Type(), stateKey, e) if req.ReplaceWholeState { write.StateDelta = upsertEntry(write.StateDelta, entry) @@ -357,8 +397,9 @@ func (r *API) persist(ctx context.Context, req persistRequest) error { // The denormalised listing columns are recomputed only when this event // is one of the ones behind them, which keeps a room full of messages - // from rewriting its own row on every line of chat. - if isState && affectsRoomProfile(e.Type()) { + // from rewriting its own row on every line of chat. A whole-state + // write may have changed any of them, so it always recomputes. + if (isState && affectsRoomProfile(e.Type())) || write.StateDeltaIsWholeState { profile := roomProfile(stateAfter) write.Profile = &profile } @@ -383,7 +424,9 @@ func (r *API) persist(ctx context.Context, req persistRequest) error { RoomID: req.Room.ID, Event: e, StreamOrdering: ordering, - StateChanged: isState, + StateChanged: stateChanged, + Origin: req.Origin, + Relay: req.Relay, }) } return nil diff --git a/server/internal/roomserver/membership.go b/server/internal/roomserver/membership.go index aa4d1b1..f6462ad 100644 --- a/server/internal/roomserver/membership.go +++ b/server/internal/roomserver/membership.go @@ -30,31 +30,25 @@ type JoinRequest struct { ServerNames []string } -// JoinRoom joins a local room. +// JoinRoom joins a room. // -// Joining a room this server does not have is a federated join: the whole -// room has to be fetched from a server that does have it, which is Phase 5. -// While federation is off that is refused with a message saying so, which is -// the behaviour PLAN.md section 3 asks for — a client otherwise sees a bare +// Joining a room this server is not in is a federated join: the room has +// to be fetched from a server that is (see federated_membership.go). While +// federation is off that is refused with a message saying so, which is the +// behaviour PLAN.md section 3 asks for — a client otherwise sees a bare // M_FORBIDDEN and the operator goes looking through power levels. func (r *API) JoinRoom(ctx context.Context, req JoinRequest) (string, error) { if !r.isLocal(req.User) { return "", fmt.Errorf("%w: %s is not a user of this server", ErrForbidden, req.User) } - switch _, err := r.store.Room(ctx, req.RoomID); { - case errors.Is(err, ErrNoSuchRoom): - if !r.federationEnabled() { - return "", fmt.Errorf("%w: %s is not a room on this server, and joining a remote room", - ErrFederationDisabled, req.RoomID) - } - // With federation on, a room we do not have still has to be - // fetched from a server that does. That is Phase 5. - return "", fmt.Errorf("%w: joining a room this server does not have is not implemented yet", - ErrFederationDisabled) - case err != nil: + remote, err := r.joinNeedsFederation(ctx, req) + if err != nil { return "", err } + if remote { + return r.joinRemote(ctx, req) + } // Joining a room you are already in is a no-op rather than a second // event: clients call /join to make sure, and every extra membership @@ -241,7 +235,31 @@ func (r *API) localUserWhoCanInvite(st auth.State, roomID string) (string, error // the user themselves. Whether they were joined, invited or knocking is the // authorisation rules' business, and a user who was never in the room is // refused by them. +// +// A room this server is not in — one it only holds an invite to — cannot +// take an event from here, and the leave goes through a server that is in +// it instead. func (r *API) LeaveRoom(ctx context.Context, roomID, userID, reason string) error { + if r.isLocal(userID) { + if _, err := r.store.Room(ctx, roomID); err != nil { + return err + } + if _, err := r.store.CurrentStateEvent(ctx, roomID, event.TypeCreate, ""); isNotFound(err) { + return r.leaveRemote(ctx, roomID, userID, reason) + } else if err != nil { + return err + } + in, err := r.participating(ctx, roomID) + if err != nil { + return err + } + if !in { + if membership, err := r.store.Membership(ctx, roomID, userID); err == nil && + membership.Membership == event.MembershipInvite && !r.isLocalEvent(ctx, roomID, membership.EventID) { + return r.leaveRemote(ctx, roomID, userID, reason) + } + } + } _, err := r.send(ctx, sendRequest{ RoomID: roomID, Type: event.TypeMember, @@ -279,12 +297,7 @@ func (r *API) InviteUser(ctx context.Context, req InviteRequest) error { return err } if !r.isLocal(req.Target) { - if !r.federationEnabled() { - return fmt.Errorf("%w: %s is on another server, and inviting a remote user", - ErrFederationDisabled, req.Target) - } - return fmt.Errorf("%w: inviting a user of another server is not implemented yet", - ErrFederationDisabled) + return r.inviteRemote(ctx, req) } _, err := r.send(ctx, sendRequest{ @@ -457,3 +470,10 @@ func validateUserID(userID string) error { } return nil } + +// isLocalEvent reports whether an event of a room was sent by a user of +// this server. +func (r *API) isLocalEvent(ctx context.Context, roomID, eventID string) bool { + stored, err := r.store.Event(ctx, roomID, eventID) + return err == nil && r.isLocal(stored.Sender) +} diff --git a/server/internal/roomserver/remote.go b/server/internal/roomserver/remote.go new file mode 100644 index 0000000..93d737f --- /dev/null +++ b/server/internal/roomserver/remote.go @@ -0,0 +1,335 @@ +package roomserver + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "slices" + "sort" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/auth" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" + "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// RemoteServers is what the roomserver asks of other servers. +// +// *federation.Client satisfies it. It is an interface so that the +// roomserver's federation paths can be tested against a fake server in +// the same process, and so that the roomserver depends on the shape of the +// federation API rather than on how requests are signed and routed. +type RemoteServers interface { + MakeJoin(ctx context.Context, destination, roomID, userID string, versions []string) (federation.MakeJoinResponse, error) + SendJoin(ctx context.Context, destination, roomID, eventID string, joinEvent json.RawMessage) (federation.SendJoinResponse, error) + MakeLeave(ctx context.Context, destination, roomID, userID string) (federation.MakeJoinResponse, error) + SendLeave(ctx context.Context, destination, roomID, eventID string, leaveEvent json.RawMessage) error + Invite(ctx context.Context, destination, roomID, eventID string, req federation.InviteRequest) (federation.InviteResponse, error) + Event(ctx context.Context, destination, eventID string) (federation.Transaction, error) + EventAuth(ctx context.Context, destination, roomID, eventID string) (federation.EventAuthResponse, error) + StateIDs(ctx context.Context, destination, roomID, eventID string) (federation.StateIDsResponse, error) + State(ctx context.Context, destination, roomID, eventID string) (federation.StateResponse, error) + MissingEvents(ctx context.Context, destination, roomID string, req federation.MissingEventsRequest) (federation.MissingEventsResponse, error) + Backfill(ctx context.Context, destination, roomID string, from []string, limit int) (federation.Transaction, error) + QueryDirectory(ctx context.Context, destination, alias string) (federation.DirectoryResponse, error) + PublicRooms(ctx context.Context, destination string, limit int, since, search string) (json.RawMessage, error) +} + +// federating reports whether this server may talk to others right now. +func (r *API) federating() bool { + return r.remote != nil && r.keys != nil && r.federationEnabled() +} + +// requireFederation is the error for an operation that needs another +// server: the switch is off, or this build has no federation at all. +func (r *API) requireFederation(action string) error { + if r.federating() { + return nil + } + return fmt.Errorf("%w: %s", ErrFederationDisabled, action) +} + +// participating reports whether this server is in a room: it holds the +// room's state, and at least one of its users is joined. +// +// Holding the room row is not enough. An invite to a room nobody here is +// in leaves a row with one membership and no create event, and a room +// every local user has left is a room whose state stopped being kept up +// to date the moment the last of them went. +func (r *API) participating(ctx context.Context, roomID string) (bool, error) { + if _, err := r.store.CurrentStateEvent(ctx, roomID, event.TypeCreate, ""); err != nil { + if isNotFound(err) { + return false, nil + } + return false, err + } + return r.store.HasJoinedMemberOn(ctx, roomID, r.serverName) +} + +// JoinedServers lists the servers with a user joined to a room, this one +// included when it is, sorted. It is who an event sent into the room is +// delivered to. +func (r *API) JoinedServers(ctx context.Context, roomID string) ([]string, error) { + joined, err := r.store.MembershipEntries(ctx, roomID, []string{event.MembershipJoin}) + if err != nil { + return nil, err + } + seen := map[string]bool{} + for _, m := range joined { + if server, err := id.ServerNameOf(m.UserID); err == nil { + seen[server] = true + } + } + out := make([]string, 0, len(seen)) + for server := range seen { + out = append(out, server) + } + sort.Strings(out) + return out, nil +} + +// ServerInRoom reports whether a server has a user joined to a room. +func (r *API) ServerInRoom(ctx context.Context, roomID, serverName string) (bool, error) { + return r.store.HasJoinedMemberOn(ctx, roomID, serverName) +} + +// candidateServers orders the servers worth asking about a room: the ones +// the caller suggested, then the one named in the room ID, then the +// servers of the room's members as this server last knew them. This +// server is never among them. +func (r *API) candidateServers(ctx context.Context, roomID string, hints []string) []string { + var out []string + add := func(server string) { + if server != "" && server != r.serverName && !slices.Contains(out, server) { + out = append(out, server) + } + } + for _, h := range hints { + add(h) + } + if server, err := id.ServerNameOfRoomID(roomID); err == nil { + add(server) + } + if members, err := r.store.MembershipEntries(ctx, roomID, nil); err == nil { + // Joined members first: they are in the room, and an invited or + // departed one may not be. + for _, m := range members { + if m.Membership == event.MembershipJoin { + if server, err := id.ServerNameOf(m.UserID); err == nil { + add(server) + } + } + } + for _, m := range members { + if server, err := id.ServerNameOf(m.UserID); err == nil { + add(server) + } + // The sender of an invite to a local user is who brought this + // server to the room, and is the server most likely to let it + // in. + if m.Membership == event.MembershipInvite && r.isLocal(m.UserID) { + if stored, err := r.store.Event(ctx, roomID, m.EventID); err == nil { + if server, err := id.ServerNameOf(stored.Sender); err == nil { + add(server) + } + } + } + } + } + return out +} + +// remoteError translates a failure from another server into the +// roomserver's own sentinels, so that a client hears "forbidden" when the +// remote server said forbidden, rather than "a remote server failed". +func remoteError(server string, err error) error { + var httpErr *federation.HTTPError + if errors.As(err, &httpErr) { + switch { + case httpErr.Code == "M_INCOMPATIBLE_ROOM_VERSION": + return fmt.Errorf("%w: %s: %s", ErrIncompatibleRoomVersion, server, httpErr.Message) + case httpErr.Status == http.StatusForbidden: + return fmt.Errorf("%w: %s refused: %s", ErrForbidden, server, httpErr.Message) + case httpErr.Status == http.StatusNotFound: + return fmt.Errorf("%w: %s does not know the room: %s", ErrNoSuchRoom, server, httpErr.Message) + } + } + switch { + case errors.Is(err, federation.ErrDisabled): + return fmt.Errorf("%w: %v", ErrFederationDisabled, err) + case errors.Is(err, federation.ErrNotAllowed): + return fmt.Errorf("%w: %v", ErrForbidden, err) + } + return fmt.Errorf("%w: %s: %v", ErrRemote, server, err) +} + +// parseRemote parses an event received from another server under the +// rules of the room it claims to belong to. +func parseRemote(raw json.RawMessage, rules roomversion.Rules) (*event.Event, error) { + e, err := event.Parse(stripUnsigned(raw), rules) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrBadEvent, err) + } + return e, nil +} + +// stripUnsigned drops the unsigned block of an event received from another +// server. It is that server's annotation — age, the state it replaced, the +// invite room state — and anything kept from it would be served to our +// own clients as if this server had written it. +func stripUnsigned(raw json.RawMessage) json.RawMessage { + var obj map[string]json.RawMessage + if err := json.Unmarshal(raw, &obj); err != nil { + return raw + } + if _, ok := obj["unsigned"]; !ok { + return raw + } + delete(obj, "unsigned") + out, err := json.Marshal(obj) + if err != nil { + return raw + } + return out +} + +// roomIDOfRaw reads the room_id of an event that has not been parsed yet, +// which is what decides the rules it will be parsed under. +func roomIDOfRaw(raw json.RawMessage) (string, error) { + var probe struct { + RoomID string `json:"room_id"` + } + if err := json.Unmarshal(raw, &probe); err != nil { + return "", fmt.Errorf("%w: %v", ErrBadEvent, err) + } + if probe.RoomID == "" { + return "", fmt.Errorf("%w: no room_id", ErrBadEvent) + } + return probe.RoomID, nil +} + +// storedOf projects an event onto the row the store keeps. +func storedOf(roomID string, e *event.Event) storage.StoredEvent { + s := storage.StoredEvent{ + RoomID: roomID, + EventID: e.EventID(), + Type: e.Type(), + Sender: e.Sender(), + Depth: e.Depth(), + OriginServerTS: e.OriginServerTS(), + JSON: e.JSON(), + } + if key, ok := e.StateKey(); ok { + s.StateKey = &key + } + return s +} + +// supportedVersionStrings lists the room versions Warp can join, for the +// ?ver= parameters of make_join. +func supportedVersionStrings() []string { + versions := roomversion.Supported() + out := make([]string, 0, len(versions)) + for _, v := range versions { + out = append(out, string(v)) + } + return out +} + +// eventsByDepth sorts events oldest first, with the event ID as the tie +// break so that the order is the same on every run. +func eventsByDepth(events []*event.Event) { + sort.SliceStable(events, func(i, j int) bool { + if events[i].Depth() != events[j].Depth() { + return events[i].Depth() < events[j].Depth() + } + return events[i].EventID() < events[j].EventID() + }) +} + +// authCheckChain checks a set of events from another server against their +// own auth events, oldest first, and returns the ones that pass. +// +// It is how a room's state and auth chain, as handed over by a join or a +// /state fetch, is judged: nothing in it is trusted because the resident +// server sent it, and an event whose auth events failed fails with them. +// known supplies auth events from outside the set — ones this server +// already holds. +func (r *API) authCheckChain(ctx context.Context, events []*event.Event, known map[string]*event.Event) (map[string]*event.Event, []string) { + eventsByDepth(events) + passed := map[string]*event.Event{} + var rejected []string + + lookup := func(id string) *event.Event { + if e, ok := passed[id]; ok { + return e + } + return known[id] + } + + // The create event is found first: from room version 12 nobody cites + // it, and every rule needs it. + var create *event.Event + for _, e := range events { + if e.Type() == event.TypeCreate { + create = e + break + } + } + if create == nil { + for _, e := range known { + if e.Type() == event.TypeCreate { + create = e + break + } + } + } + + for _, e := range events { + var authEvents []*event.Event + missing := false + for _, id := range e.AuthEventIDs() { + ae := lookup(id) + if ae == nil { + missing = true + break + } + authEvents = append(authEvents, ae) + } + if missing { + rejected = append(rejected, e.EventID()) + continue + } + if err := auth.CheckAuthEvents(e, authEvents); err != nil { + rejected = append(rejected, e.EventID()) + continue + } + st := auth.NewState(authEvents) + if st.Create() == nil && create != nil { + st[auth.StateKey{Type: event.TypeCreate}] = create + } + if err := auth.Check(ctx, e, st, r.verifier()); err != nil { + r.log.Debug("event in a received chain fails authorisation", + "event_id", e.EventID(), "type", e.Type(), "error", err) + rejected = append(rejected, e.EventID()) + continue + } + passed[e.EventID()] = e + } + return passed, rejected +} + +// inRoom reports whether an event belongs to a room. A create event of +// room version 12 carries no room_id; it belongs to the room its own ID +// names. +func inRoom(e *event.Event, roomID string) bool { + if e.Type() == event.TypeCreate && e.Rules().RoomIDIsCreateEventID { + id, err := event.RoomIDOfCreateEvent(e) + return err == nil && id == roomID + } + return e.RoomID() == roomID +} diff --git a/server/internal/roomserver/roomserver.go b/server/internal/roomserver/roomserver.go index 211ebb7..01e6a29 100644 --- a/server/internal/roomserver/roomserver.go +++ b/server/internal/roomserver/roomserver.go @@ -66,6 +66,23 @@ var ( // ErrUnknownRoomVersion is a room version that does not exist. ErrUnknownRoomVersion = roomversion.ErrUnknownVersion + + // ErrIncompatibleRoomVersion is a federated join or invite to a room + // whose version the other side does not support. + ErrIncompatibleRoomVersion = errors.New("incompatible room version") + + // ErrRemote is another server failing to do what was asked of it: + // unreachable, refusing, or answering with something unusable. The + // wrapped message says which server and why. + ErrRemote = errors.New("remote server failed") + + // ErrNotInRoom is a federation request about a room this server is + // not in, or from a server that is not in it. + ErrNotInRoom = errors.New("not in room") + + // ErrBadEvent is an event from another server that is malformed or + // does not verify. + ErrBadEvent = errors.New("bad event") ) // Profiles supplies the display name and avatar that a membership event @@ -92,6 +109,19 @@ type Notification struct { // StateChanged reports whether the room's current state moved. StateChanged bool + + // Origin is the server the event was received from, or empty for an + // event created here. + Origin string + + // Relay asks for an event received from Origin to be passed on to the + // room's other servers. Only a join, leave or knock handed to this + // server through send_join, send_leave or send_knock is relayed: the + // server that sent it is not yet (or no longer) in the room, so it + // relies on the resident server to tell everybody else. An event + // received over /send is never relayed, because its origin sends it to + // every server in the room itself. + Relay bool } // Notifier is told about every event this server persists, in the order @@ -129,12 +159,21 @@ type API struct { // ask for one. defaultRoomVersion roomversion.Version + // remote is the federation client and keys verifies other servers' + // signatures. Both are nil in a build with no federation, which then + // behaves as one with the switch permanently off. + remote RemoteServers + keys auth.Verifier + // locks serialises the writers of each room. Two events racing into one // room would each pick up the same forward extremities and both become // one, forking the room for no reason — and then needing state // resolution to put it back together. Taking a lock per room instead // keeps every locally created event a child of the one before it. locks sync.Map // room ID -> *sync.Mutex + + // backfills serialises the backfills of each room; see backfillLock. + backfills sync.Map // room ID -> *sync.Mutex } // Options configure the component. @@ -164,6 +203,13 @@ type Options struct { // Notifier receives every persisted event. Nil skips the emit stage. Notifier Notifier + // Remote is the federation client: joins, leaves, invites and the + // fetches that fill in a room's history. Nil means no federation. + Remote RemoteServers + + // Keys verifies other servers' signatures. Nil means no federation. + Keys auth.Verifier + Logger *slog.Logger // Now is the clock, so that tests can stamp events deterministically. @@ -211,6 +257,8 @@ func New(opts Options) (*API, error) { now: opts.Now, federationEnabled: opts.FederationEnabled, defaultRoomVersion: version, + remote: opts.Remote, + keys: opts.Keys, }, nil } diff --git a/server/internal/roomserver/roomsync.go b/server/internal/roomserver/roomsync.go index ad92622..9c35ee0 100644 --- a/server/internal/roomserver/roomsync.go +++ b/server/internal/roomserver/roomsync.go @@ -175,13 +175,16 @@ func (r *API) syncState(ctx context.Context, rules roomversion.Rules, visible Vi // timeline events in this window is reported as of the top of the sync // instead: the state may still have been read for the first time. func statePosition(timeline Timeline, to int64, visible VisibleRange) (int64, bool) { + // Zero is the one position nothing can be at, so it stands for "before + // the room began"; a negative Start is inside backfilled history and + // has state like any other position. if len(timeline.Events) > 0 { - return timeline.Start, timeline.Start > 0 + return timeline.Start, timeline.Start != 0 } if visible.To != 0 && (to == 0 || to > visible.To) { to = visible.To } - return to, to > 0 + return to, to != 0 } // shapeState applies the state filter and lazy loading to a state set. diff --git a/server/internal/roomserver/stream.go b/server/internal/roomserver/stream.go index b5bc5bc..56dd4a8 100644 --- a/server/internal/roomserver/stream.go +++ b/server/internal/roomserver/stream.go @@ -36,7 +36,8 @@ type MembershipAt struct { // TimelineRequest is one window of a room's event stream. // // The bounds are the store's: Low exclusive, High inclusive, both -// directions, and zero meaning unbounded at that end. Direction decides +// directions, and zero meaning unbounded at that end. Either can be +// negative, inside history backfilled from other servers. Direction decides // only which end the limit trims and the order the store reads in — the // events always come back oldest first. type TimelineRequest struct { @@ -59,6 +60,13 @@ type TimelineRequest struct { // after the limit is what keeps a filtered page from coming back short // and looking like the end of the room. Filter storage.TimelineFilter + + // Backfill allows a backwards read that runs out of history at the + // bottom of what this server holds to ask the servers in the room for + // more before it answers (see backfill.go). /messages sets it; /sync + // and /context do not, because neither is a client asking to scroll + // back, and neither should wait on another server. + Backfill bool } // Timeline is a window of a room's event stream. @@ -121,7 +129,35 @@ func (r *API) Timeline(ctx context.Context, req TimelineRequest) (Timeline, erro if err != nil { return Timeline{}, err } - return r.timeline(ctx, rules, visible, req) + page, err := r.timeline(ctx, rules, visible, req) + if err != nil { + return Timeline{}, err + } + + // A page that came back short at the bottom of the room is either the + // beginning of the room or the beginning of what this server holds of + // it. Telling a client it has reached the end — no `end` token — when + // it is the second would stop it scrolling for good, so the servers in + // the room are asked first. + for range backfillRounds { + if !r.wantsBackfill(req, visible, page) { + break + } + added, err := r.backfill(ctx, room, rules, req.Limit) + if err != nil { + // The page this server has is still an answer. The client + // will try again the next time it scrolls up. + r.log.Info("could not backfill a room", "room_id", room.ID, "error", err) + break + } + if added == 0 { + break + } + if page, err = r.timeline(ctx, rules, visible, req); err != nil { + return Timeline{}, err + } + } + return page, nil } // timeline is Timeline with the room's rules and the user's visible range @@ -130,7 +166,7 @@ func (r *API) timeline(ctx context.Context, rules roomversion.Rules, visible Vis // An empty window is reported at the bound the caller came in on, so // that the tokens it hands back are still usable. empty := Timeline{Start: req.Low, End: req.Low} - if req.Direction == storage.DirectionBackward && req.High > 0 { + if req.Direction == storage.DirectionBackward && req.High != 0 { empty = Timeline{Start: req.High, End: req.High} } if req.Limit <= 0 { @@ -181,8 +217,10 @@ func (r *API) timeline(ctx context.Context, rules roomversion.Rules, visible Vis } // Start sits below the oldest event, so that handing it back as a High // bound returns the page before this one; End is at the newest, so that - // handing it back as a Low bound returns the page after. - out.Start = stored[0].StreamOrdering - 1 + // handing it back as a Low bound returns the page after. Below the + // event at 1 is -1, not 0: zero would read as "unbounded" and hand the + // client the whole room again rather than the history beneath it. + out.Start = storage.PositionBefore(stored[0].StreamOrdering) out.End = stored[len(stored)-1].StreamOrdering return out, nil } @@ -359,14 +397,62 @@ var strippedStateTypes = []string{ // The member events are what makes an invite legible: a client showing // "Alice invited you" needs Alice's display name, and the invitee's own // member event carries the invite's reason. +// +// For a room this server is not in — an invite that came over federation — +// the room's state is not here, and what is returned is the stripped state +// the inviting server sent along, plus the invite itself. func (r *API) StrippedState(ctx context.Context, roomID string, members []string) ([]StrippedEvent, error) { _, rules, err := r.roomAndRules(ctx, roomID) if err != nil { return nil, err } + if _, err := r.store.CurrentStateEvent(ctx, roomID, event.TypeCreate, ""); isNotFound(err) { + return r.outOfBandStrippedState(ctx, roomID, rules, members) + } else if err != nil { + return nil, err + } return r.strippedState(ctx, roomID, rules, members) } +// outOfBandStrippedState is StrippedState for a room known only from an +// invite. +func (r *API) outOfBandStrippedState(ctx context.Context, roomID string, rules roomversion.Rules, members []string) ([]StrippedEvent, error) { + var out []StrippedEvent + for _, member := range members { + raw, err := r.store.InviteState(ctx, roomID, member) + if err != nil { + return nil, err + } + if len(raw) > 0 { + var stripped []StrippedEvent + if err := json.Unmarshal(raw, &stripped); err != nil { + // What another server sent is not ours to fail a sync over. + r.log.Warn("stored invite state is unreadable", "room_id", roomID, "user_id", member, "error", err) + } + for _, s := range stripped { + if s.Type == event.TypeMember && s.StateKey == member { + continue // the invite itself is added below + } + out = append(out, s) + } + } + } + for _, member := range members { + stored, err := r.store.CurrentStateEvent(ctx, roomID, event.TypeMember, member) + if isNotFound(err) { + continue + } else if err != nil { + return nil, err + } + e, err := parseEvent(stored, rules) + if err != nil { + return nil, err + } + out = append(out, StrippedEvent{Type: e.Type(), StateKey: member, Sender: e.Sender(), Content: e.Content()}) + } + return out, nil +} + // strippedState is StrippedState with the rules in hand. func (r *API) strippedState(ctx context.Context, roomID string, rules roomversion.Rules, members []string) ([]StrippedEvent, error) { var out []StrippedEvent diff --git a/server/internal/roomserver/verify.go b/server/internal/roomserver/verify.go index 345ae7f..3ce34f9 100644 --- a/server/internal/roomserver/verify.go +++ b/server/internal/roomserver/verify.go @@ -4,37 +4,73 @@ import ( "context" "fmt" + "github.com/dariuszjaniak/warp/server/internal/matrix/auth" "github.com/dariuszjaniak/warp/server/internal/matrix/event" "github.com/dariuszjaniak/warp/server/internal/matrix/signing" ) -// localVerifier checks an event's signature against this server's own key. +// eventVerifier checks an event's signature from a named server. // -// One authorisation rule needs somebody else's signature verified: a join -// under the restricted join rule is authorised by a third party named in -// content.join_authorised_via_users_server, and the event must carry a -// signature from *that user's* server. While federation is off the only -// such server that can appear is this one, and its key is already in hand — -// so the rule can be applied in full, locally, without a key fetcher. -// -// Phase 5 replaces this with a verifier that can fetch and cache another -// server's keys. Until then a join authorised via a remote user is refused -// rather than waved through, which is the same answer the federation switch +// This server's own signatures are checked against the key in hand, which +// is what keeps every rule that needs a signature — a restricted join +// authorised by a local user, chiefly — working with federation off. +// Anybody else's go to the key ring, and only while federation is on: with +// it off there is no way to fetch a key, and a signature that cannot be +// checked is refused rather than waved through, the same answer the switch // gives everywhere else. -type localVerifier struct { +type eventVerifier struct { serverName string key *signing.KeyPair + remote auth.Verifier + enabled func() bool } // verifier returns the Verifier the authorisation rules are given. -func (r *API) verifier() *localVerifier { - return &localVerifier{serverName: r.serverName, key: r.signingKey} +func (r *API) verifier() *eventVerifier { + return &eventVerifier{ + serverName: r.serverName, + key: r.signingKey, + remote: r.keys, + enabled: r.federationEnabled, + } } // VerifyEventSignature implements auth.Verifier. -func (v *localVerifier) VerifyEventSignature(_ context.Context, serverName string, e *event.Event) error { - if serverName != v.serverName { +func (v *eventVerifier) VerifyEventSignature(ctx context.Context, serverName string, e *event.Event) error { + if serverName == v.serverName { + return e.VerifySignature(v.serverName, v.key.KeyID(), v.key.Public) + } + if v.remote == nil || !v.enabled() { return fmt.Errorf("%w: cannot verify a signature from %s", ErrFederationDisabled, serverName) } - return e.VerifySignature(v.serverName, v.key.KeyID(), v.key.Public) + return v.remote.VerifyEventSignature(ctx, serverName, e) +} + +// verifyIncoming checks an event from another server before anything else +// looks at it: it must be signed by its sender's server, and its content +// must match its hash. +// +// A content hash mismatch is not a rejection. The signature covers only +// what a redaction keeps, so an event whose signature verifies and whose +// hash does not is an event somebody has redacted — legitimately, or by +// tampering with the parts the signature does not cover — and the spec's +// answer to both is the same: keep the redacted form. +func (r *API) verifyIncoming(ctx context.Context, e *event.Event) (*event.Event, error) { + sender, err := e.SenderServer() + if err != nil { + return nil, fmt.Errorf("%w: %s: %v", ErrBadEvent, e.EventID(), err) + } + if err := r.verifier().VerifyEventSignature(ctx, sender, e); err != nil { + return nil, fmt.Errorf("%w: %s is not signed by %s: %v", ErrBadEvent, e.EventID(), sender, err) + } + if err := e.VerifyContentHash(); err != nil { + redacted, rerr := e.Redact() + if rerr != nil { + return nil, fmt.Errorf("%w: %s: %v", ErrBadEvent, e.EventID(), rerr) + } + r.log.Info("event failed its content hash; using its redacted form", + "event_id", e.EventID(), "sender", e.Sender()) + return redacted, nil + } + return e, nil } diff --git a/server/internal/roomserver/visibility.go b/server/internal/roomserver/visibility.go index 0e9d0dd..b0a0926 100644 --- a/server/internal/roomserver/visibility.go +++ b/server/internal/roomserver/visibility.go @@ -6,6 +6,7 @@ import ( "github.com/dariuszjaniak/warp/server/internal/matrix/event" "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/storage" ) // VisibleRange is the span of a room's event stream a user may read. @@ -22,7 +23,8 @@ type VisibleRange struct { Allowed bool // From is the first position the user may see, exclusive. Zero means - // the start of the room. + // the start of the room, backfilled history included: positions can be + // negative, so zero is the one value that cannot be a real floor. From int64 // To is the last position the user may see, inclusive. Zero means the @@ -43,7 +45,7 @@ func (v VisibleRange) Contains(pos int64) bool { switch { case !v.Allowed: return false - case pos <= v.From: + case v.From != 0 && pos <= v.From: return false case v.To != 0 && pos > v.To: return false @@ -61,11 +63,16 @@ func (v VisibleRange) Clamp(from, to int64) (int64, int64, bool) { if !v.Allowed { return 0, 0, false } - from = max(from, v.From) + // Zero is unbounded at either end, and every other value — negative + // ones inside backfilled history among them — is a real bound, so the + // narrower of two bounds is not simply the larger or smaller number. + if v.From != 0 && (from == 0 || from < v.From) { + from = v.From + } if v.To != 0 && (to == 0 || to > v.To) { to = v.To } - if to != 0 && from >= to { + if from != 0 && to != 0 && from >= to { return 0, 0, false } return from, to, true @@ -207,7 +214,7 @@ func spanStart(changes []membershipChange, historyVisibility string) int64 { } // Exclusive, so one below the event itself: the membership event is the // first thing the user sees, which is how a client renders "you joined". - return changes[start].position - 1 + return storage.PositionBefore(changes[start].position) } // requireReadable refuses a caller who may see nothing of a room, and diff --git a/server/internal/setup/federation.go b/server/internal/setup/federation.go new file mode 100644 index 0000000..1226f26 --- /dev/null +++ b/server/internal/setup/federation.go @@ -0,0 +1,77 @@ +package setup + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "os" + "sync" + + "github.com/dariuszjaniak/warp/server/internal/federationapi" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// fanout is the roomserver's emit stage when more than one thing listens: +// the sync notifier, which wakes long polls, and the federation queue, +// which sends events to other servers. +// +// Each listener is held to the roomserver's rule for a Notifier — never +// block — so calling them in turn under the room lock is fine. +type fanout struct { + mu sync.RWMutex + listeners []roomserver.Notifier +} + +func (f *fanout) add(n roomserver.Notifier) { + f.mu.Lock() + defer f.mu.Unlock() + f.listeners = append(f.listeners, n) +} + +// EventPersisted implements roomserver.Notifier. +func (f *fanout) EventPersisted(ctx context.Context, n roomserver.Notification) { + f.mu.RLock() + defer f.mu.RUnlock() + for _, l := range f.listeners { + l.EventPersisted(ctx, n) + } +} + +// federationStatus adapts the module to the admin API. +type federationStatus struct { + module *federationapi.Module +} + +func (s federationStatus) Running() bool { return s.module.Running() } + +func (s federationStatus) Destinations(ctx context.Context) ([]storage.DestinationSummary, error) { + return s.module.Queue().Destinations(ctx) +} + +func (s federationStatus) ResetBackoff(ctx context.Context, serverName string) error { + return s.module.Queue().ResetBackoff(ctx, serverName) +} + +// federationTLS builds the client TLS configuration for outbound +// federation: the system roots, plus a configured CA bundle. +func federationTLS(caFile string) (*tls.Config, error) { + cfg := &tls.Config{MinVersion: tls.VersionTLS12} + if caFile == "" { + return cfg, nil + } + pool, err := x509.SystemCertPool() + if err != nil || pool == nil { + pool = x509.NewCertPool() + } + pem, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("read federation.tls_ca_file: %w", err) + } + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("federation.tls_ca_file %s contains no PEM certificates", caFile) + } + cfg.RootCAs = pool + return cfg, nil +} diff --git a/server/internal/setup/phase5_test.go b/server/internal/setup/phase5_test.go new file mode 100644 index 0000000..f361b9b --- /dev/null +++ b/server/internal/setup/phase5_test.go @@ -0,0 +1,441 @@ +package setup_test + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "log/slog" + "math/big" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/dariuszjaniak/warp/server/internal/config" + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" + "github.com/dariuszjaniak/warp/server/internal/setup" + "github.com/dariuszjaniak/warp/server/internal/userapi" +) + +// The end-to-end federation tests: complete servers, each behind its own +// TLS listener, finding each other through a resolver that maps their +// names to those listeners. Everything between the client API on one +// server and the client API on the other is real — X-Matrix signing, key +// fetches, the routes, the gate, the outbound queue. + +// fedNode is one server of an end-to-end federation. +type fedNode struct { + name string + srv *setup.Server + ts *httptest.Server + + // handler is the server's handler, set once it exists. The listener + // has to be up before the server is built — every server needs every + // other's address — so it starts out answering 503. + handler atomic.Pointer[http.Handler] +} + +func (n *fedNode) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h := n.handler.Load() + if h == nil { + http.Error(w, "not ready", http.StatusServiceUnavailable) + return + } + (*h).ServeHTTP(w, r) +} + +// newFederation starts servers that can reach each other. +func newFederation(t *testing.T, names ...string) map[string]*fedNode { + t.Helper() + nodes := map[string]*fedNode{} + for _, name := range names { + node := &fedNode{name: name} + node.ts = httptest.NewUnstartedServer(node) + node.ts.StartTLS() + t.Cleanup(node.ts.Close) + nodes[name] = node + } + + // Every test listener has the same certificate, for "example.com"; the + // Host is that name with the listener's port, and the address is the + // listener. That is the delegated-by-SRV shape: TLS is checked against + // one name and the connection goes somewhere else. + resolve := func(_ context.Context, name string) (federation.Resolved, error) { + node, ok := nodes[name] + if !ok { + return federation.Resolved{}, &net.DNSError{Err: "no such host", Name: name, IsNotFound: true} + } + addr := node.ts.Listener.Addr().String() + _, port, _ := net.SplitHostPort(addr) + return federation.Resolved{Addresses: []string{addr}, Host: "example.com:" + port}, nil + } + anyTS := nodes[names[0]].ts + clientTLS := anyTS.Client().Transport.(*http.Transport).TLSClientConfig.Clone() + + for _, name := range names { + node := nodes[name] + key, err := signing.Generate("") + if err != nil { + t.Fatal(err) + } + keyPath := filepath.Join(t.TempDir(), name+".key") + if err := key.Save(keyPath); err != nil { + t.Fatal(err) + } + cfg := config.Default() + cfg.Global.ServerName = name + cfg.Global.PrivateKeyPath = keyPath + cfg.Database.ConnectionString = "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), name+".db")) + + "?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)" + cfg.Registration.Enabled = true + cfg.ClientAPI.RateLimit.Enabled = false + cfg.Federation.Enabled = true + if err := cfg.Validate(); err != nil { + t.Fatalf("config: %v", err) + } + srv, err := setup.New(t.Context(), cfg, setup.Options{ + Logger: slog.New(slog.DiscardHandler), + FederationResolve: resolve, + FederationTLS: clientTLS, + }) + if err != nil { + t.Fatalf("setup.New(%s): %v", name, err) + } + t.Cleanup(func() { _ = srv.Close() }) + node.srv = srv + h := srv.Handler() + node.handler.Store(&h) + t.Cleanup(srv.FederationModule().Stop) + if err := srv.FederationModule().Start(t.Context()); err != nil { + t.Fatalf("start federation on %s: %v", name, err) + } + } + return nodes +} + +// eventually polls until check passes or the deadline is reached. +func eventually(t *testing.T, what string, check func() bool) { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if check() { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +// messageBodies reads the message bodies of a room's recent history. +func messageBodies(t *testing.T, srv *setup.Server, roomID, token string) []string { + t.Helper() + rec := getWithToken(t, srv, "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/messages?dir=b&limit=50", token) + if rec.Code != http.StatusOK { + return nil + } + var resp struct { + Chunk []struct { + Type string `json:"type"` + Content struct { + Body string `json:"body"` + } `json:"content"` + } `json:"chunk"` + } + _ = json.Unmarshal(rec.Body.Bytes(), &resp) + var out []string + for _, e := range resp.Chunk { + if e.Type == "m.room.message" { + out = append(out, e.Content.Body) + } + } + return out +} + +func contains(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + +func sendMessage(t *testing.T, srv *setup.Server, roomID, token, txn, body string) { + t.Helper() + rec := put(t, srv, "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/send/m.room.message/"+txn, + map[string]any{"msgtype": "m.text", "body": body}, token) + if rec.Code != http.StatusOK { + t.Fatalf("send %q: status %d: %s", body, rec.Code, rec.Body.String()) + } +} + +func TestFederationEndToEnd(t *testing.T) { + if testing.Short() { + t.Skip("starts two servers") + } + nodes := newFederation(t, "hs-a.example", "hs-b.example") + a, b := nodes["hs-a.example"].srv, nodes["hs-b.example"].srv + + aliceToken := registerThrough(t, a, "alice") + bobToken := registerThrough(t, b, "bob") + + rec := post(t, a, "/_matrix/client/v3/createRoom", map[string]any{ + "preset": "public_chat", "name": "Across", "room_alias_name": "across", + }, aliceToken) + if rec.Code != http.StatusOK { + t.Fatalf("createRoom: %d %s", rec.Code, rec.Body.String()) + } + roomID, _ := decode[map[string]any](t, rec)["room_id"].(string) + + // Bob joins by alias: the alias is resolved by asking A, then the join + // goes through A. + rec = post(t, b, "/_matrix/client/v3/join/"+url.PathEscape("#across:hs-a.example"), map[string]any{}, bobToken) + if rec.Code != http.StatusOK { + t.Fatalf("remote join: %d %s", rec.Code, rec.Body.String()) + } + if got, _ := decode[map[string]any](t, rec)["room_id"].(string); got != roomID { + t.Fatalf("joined %q, want %q", got, roomID) + } + + // Alice hears of the join without any queue involved: it went + // through her server. + rec = getWithToken(t, a, "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/joined_members", aliceToken) + if !strings.Contains(rec.Body.String(), "@bob:hs-b.example") { + t.Fatalf("A's joined members lack bob: %s", rec.Body.String()) + } + + // Messages in both directions go through the queues. + sendMessage(t, a, roomID, aliceToken, "a1", "hello from A") + eventually(t, "A's message on B", func() bool { + return contains(messageBodies(t, b, roomID, bobToken), "hello from A") + }) + sendMessage(t, b, roomID, bobToken, "b1", "hello from B") + eventually(t, "B's message on A", func() bool { + return contains(messageBodies(t, a, roomID, aliceToken), "hello from B") + }) + + // Remote profile lookup. + rec = getWithToken(t, b, "/_matrix/client/v3/profile/@alice:hs-a.example", bobToken) + if rec.Code != http.StatusOK { + t.Errorf("remote profile: %d %s", rec.Code, rec.Body.String()) + } + + // Switching A's federation off: B's next message cannot be delivered, + // and B records the failure. Switching it back on and clearing the + // backoff delivers it. + if err := a.Settings().SetFederationEnabled(t.Context(), false); err != nil { + t.Fatal(err) + } + a.FederationModule().Stop() + sendMessage(t, b, roomID, bobToken, "b2", "while A is away") + eventually(t, "B to record A as failing", func() bool { + ds, err := b.FederationModule().Queue().Destinations(t.Context()) + if err != nil { + return false + } + for _, d := range ds { + if d.ServerName == "hs-a.example" && d.Failures > 0 && d.QueuedPDUs > 0 { + return true + } + } + return false + }) + if contains(messageBodies(t, a, roomID, aliceToken), "while A is away") { + t.Fatal("a message reached A while its federation was off") + } + + // The admin API shows the backlog and the backoff. + root, err := b.Users().CreateAccount(t.Context(), userapi.NewAccountRequest{ + Localpart: "root", Password: "sUp3rs3kr1t", Admin: true, + }) + if err != nil { + t.Fatal(err) + } + session, err := b.Users().CreateSession(t.Context(), userapi.SessionRequest{Localpart: root.Localpart}) + if err != nil { + t.Fatal(err) + } + rec = getWithToken(t, b, "/_warp/admin/v1/federation/destinations", session.AccessToken) + if rec.Code != http.StatusOK { + t.Fatalf("GET destinations: %d %s", rec.Code, rec.Body.String()) + } + var listing struct { + Running bool `json:"running"` + Destinations []struct { + ServerName string `json:"server_name"` + Failures int `json:"failures"` + QueuedPDUs int `json:"queued_pdus"` + } `json:"destinations"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &listing); err != nil { + t.Fatal(err) + } + if !listing.Running || len(listing.Destinations) == 0 || + listing.Destinations[0].ServerName != "hs-a.example" || listing.Destinations[0].Failures == 0 { + t.Errorf("destinations = %s", rec.Body.String()) + } + + if err := a.Settings().SetFederationEnabled(t.Context(), true); err != nil { + t.Fatal(err) + } + if err := a.FederationModule().Start(t.Context()); err != nil { + t.Fatal(err) + } + if err := b.FederationModule().Queue().ResetBackoff(t.Context(), "hs-a.example"); err != nil { + t.Fatal(err) + } + eventually(t, "the held message on A", func() bool { + return contains(messageBodies(t, a, roomID, aliceToken), "while A is away") + }) +} + +func TestFederationEndpointsAreGated(t *testing.T) { + srv := newSigningServer(t, nil) + for _, path := range []string{ + "/_matrix/key/v2/server", + "/_matrix/federation/v1/version", + } { + if rec := get(t, srv, http.MethodGet, path); rec.Code != http.StatusNotFound { + t.Errorf("%s with federation off = %d, want 404", path, rec.Code) + } + } + + if err := srv.Settings().SetFederationEnabled(t.Context(), true); err != nil { + t.Fatal(err) + } + rec := get(t, srv, http.MethodGet, "/_matrix/key/v2/server") + if rec.Code != http.StatusOK { + t.Fatalf("key server with federation on = %d", rec.Code) + } + if name, _ := decode[map[string]any](t, rec)["server_name"].(string); name != "warp.example" { + t.Errorf("key response server_name = %q", name) + } + + // An unsigned request to an authenticated endpoint is refused. + rec = get(t, srv, http.MethodGet, "/_matrix/federation/v1/event/$abc") + if rec.Code != http.StatusUnauthorized { + t.Errorf("unsigned /event = %d, want 401", rec.Code) + } +} + +// writeSelfSignedCert writes a certificate for 127.0.0.1 and its key, and +// returns their paths and a pool that trusts the certificate. +func writeSelfSignedCert(t *testing.T) (string, string, *x509.CertPool) { + t.Helper() + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "warp.example"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &priv.PublicKey, priv) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalECPrivateKey(priv) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + certPath, keyPath := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem") + if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil { + t.Fatal(err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + pool.AddCert(cert) + return certPath, keyPath, pool +} + +func TestFederationListenerServesTLS(t *testing.T) { + certPath, keyPath, pool := writeSelfSignedCert(t) + srv := newSigningServer(t, func(c *config.Config) { + c.Listen.Address = "127.0.0.1:0" + c.Listen.FederationAddress = "127.0.0.1:0" + c.Listen.TLSCertFile = certPath + c.Listen.TLSKeyFile = keyPath + c.Federation.Enabled = true + c.Database.ConnectionString = "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "warp.db")) + }) + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- srv.Run(ctx) }() + defer func() { + cancel() + <-done + }() + + var addr string + for range 200 { + if addr = srv.FederationAddr(); addr != "" { + break + } + time.Sleep(10 * time.Millisecond) + } + if addr == "" { + t.Fatal("the federation listener never started") + } + if !srv.FederationModule().Running() { + t.Error("Run did not start the federation module with federation on") + } + + client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}} + resp, err := client.Get("https://" + addr + "/_matrix/key/v2/server") + if err != nil { + t.Fatalf("GET over TLS: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("key server over TLS = %d", resp.StatusCode) + } + + // The runtime switch stops the module while Run is serving. + if err := srv.Settings().SetFederationEnabled(t.Context(), false); err != nil { + t.Fatal(err) + } + if srv.FederationModule().Running() { + t.Error("switching federation off did not stop the module") + } + if err := srv.Settings().SetFederationEnabled(t.Context(), true); err != nil { + t.Fatal(err) + } + if !srv.FederationModule().Running() { + t.Error("switching federation back on did not start the module") + } +} + +func TestFederationListenerNeedsACertificate(t *testing.T) { + cfg := config.Default() + cfg.Global.ServerName = "warp.example" + cfg.Listen.FederationAddress = ":8448" + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "tls_cert_file") { + t.Errorf("Validate = %v, want a complaint about the certificate", err) + } +} diff --git a/server/internal/setup/routes.go b/server/internal/setup/routes.go index b93c821..e43fd8c 100644 --- a/server/internal/setup/routes.go +++ b/server/internal/setup/routes.go @@ -14,10 +14,10 @@ import ( // actually uses from it work. Accounts, sessions, devices, profiles, // account data, rooms and — with Phase 4 — the event stream all do, which // is enough for a client to be used. What is still missing is end-to-end -// encryption (Phase 7), media (Phase 6), push (Phase 8) and federation -// (Phase 5); a client discovers each of those from the endpoint it needs -// rather than from this list, which is why their absence does not make a -// version listed here a lie. +// encryption (Phase 7), media (Phase 6) and push (Phase 8); a client +// discovers each of those from the endpoint it needs rather than from this +// list, which is why their absence does not make a version listed here a +// lie. var SpecVersions = []string{"v1.13", "v1.14"} // UnstableFeatures advertises MSC support. Simplified Sliding Sync @@ -62,6 +62,9 @@ func (s *Server) routes() *httputil.Router { r.Handle("GET /.well-known/matrix/server", s.federation.ProtectFunc(s.handleWellKnownServer)) s.clientAPI.RegisterRoutes(r) + if s.fedAPI != nil { + s.fedAPI.RegisterRoutes(r) + } s.adminAPI.RegisterRoutes(r) r.HandleFunc("GET /healthz", s.handleHealth) diff --git a/server/internal/setup/server.go b/server/internal/setup/server.go index dc65794..5f7bf72 100644 --- a/server/internal/setup/server.go +++ b/server/internal/setup/server.go @@ -5,6 +5,7 @@ package setup import ( "context" + "crypto/tls" "errors" "fmt" "log/slog" @@ -18,6 +19,7 @@ import ( "github.com/dariuszjaniak/warp/server/internal/clientapi" "github.com/dariuszjaniak/warp/server/internal/config" "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/federationapi" "github.com/dariuszjaniak/warp/server/internal/httputil" "github.com/dariuszjaniak/warp/server/internal/matrix/signing" "github.com/dariuszjaniak/warp/server/internal/metrics" @@ -46,11 +48,30 @@ type Server struct { clientAPI *clientapi.ClientAPI adminAPI *adminapi.AdminAPI + // The federation components. All nil when the server started without + // a signing key, because nothing that federates can work without one. + fedClient *federation.Client + keyRing *federation.KeyRing + fedModule *federationapi.Module + fedAPI *federationapi.Server + + // tlsConfig serves the federation listener; nil when there is none. + tlsConfig *tls.Config + + // running is set while Run is serving. The federation switch only + // starts and stops the module while it is: a server that was built and + // never run must not leave a queue running behind it. + running atomic.Bool + handler http.Handler // addr is the address Run is listening on; empty until then. addr atomic.Pointer[string] + // fedAddr is the TLS federation listener's address while Run is + // serving one; empty otherwise. + fedAddr atomic.Pointer[string] + // ownsDB is false when the caller supplied the database and will close // it itself. ownsDB bool @@ -69,6 +90,13 @@ type Options struct { // Only tests and `warp migrate` use it: a real server must be able to // sign. SkipSigningKey bool + + // FederationResolve replaces server discovery, and FederationTLS the + // outbound TLS configuration. Tests that run several servers in one + // process use them to point server names at local listeners whose + // certificates they trust. + FederationResolve federation.ResolveFunc + FederationTLS *tls.Config } // New builds the server: storage, migrations, settings, routes. It does not @@ -128,12 +156,24 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) settingsService.FederationEnabled, federation.NewPolicy(cfg.Federation.Allowlist, cfg.Federation.Denylist), ) - // Phase 5 replaces this with starting and stopping the federation - // module; for now it is the audit trail for a runtime toggle. + // The switch starts and stops the federation module. The gate and the + // client read the switch on every request and need no telling. settingsService.Subscribe(func(snap settings.Snapshot) { log.Info("federation switch applied", "enabled", snap.FederationEnabled) + if s.fedModule != nil && s.running.Load() { + s.fedModule.Apply(context.Background(), snap.FederationEnabled) + } }) + if cfg.Listen.FederationAddress != "" { + cert, err := tls.LoadX509KeyPair(cfg.Listen.TLSCertFile, cfg.Listen.TLSKeyFile) + if err != nil { + _ = s.closeDB() + return nil, fmt.Errorf("load the federation listener's certificate: %w", err) + } + s.tlsConfig = &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12} + } + s.users, err = userapi.New(userapi.Options{ ServerName: cfg.Global.ServerName, Store: s.db.Users(), @@ -159,8 +199,40 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) // The notifier is built first and handed to both: it is the // roomserver's emit stage and the syncapi's wake-up, which is exactly // the seam Phase 3 left for this. + // + // Federation hangs off the same key. The client and the key ring are + // built whether or not the switch is on — they refuse everything while + // it is off — and the outbound queue is a second listener on the + // roomserver's emit stage next to the notifier. if s.signingKey != nil { notifier := syncapi.NewNotifier() + emit := &fanout{} + emit.add(notifier) + + clientTLS := opts.FederationTLS + if clientTLS == nil { + if clientTLS, err = federationTLS(cfg.Federation.TLSCAFile); err != nil { + _ = s.closeDB() + return nil, err + } + } + s.fedClient = federation.NewClient(federation.ClientOptions{ + ServerName: cfg.Global.ServerName, + Key: s.signingKey, + Resolve: opts.FederationResolve, + TLSConfig: clientTLS, + Enabled: settingsService.FederationEnabled, + Policy: s.federation.Policy(), + }) + s.keyRing = federation.NewKeyRing(federation.KeyRingOptions{ + ServerName: cfg.Global.ServerName, + Key: s.signingKey, + Store: s.db.Federation(), + Client: s.fedClient, + Notaries: cfg.Federation.TrustedKeyServers, + Validity: cfg.Federation.KeyValidityPeriod, + Logger: log, + }) s.rooms, err = roomserver.New(roomserver.Options{ ServerName: cfg.Global.ServerName, @@ -168,7 +240,9 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) Profiles: s.users, SigningKey: s.signingKey, FederationEnabled: settingsService.FederationEnabled, - Notifier: notifier, + Notifier: emit, + Remote: s.fedClient, + Keys: s.keyRing, Logger: log, }) if err != nil { @@ -176,6 +250,17 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) return nil, err } + queue := federationapi.NewQueue(federationapi.QueueOptions{ + ServerName: cfg.Global.ServerName, + Store: s.db.Federation(), + Sender: s.fedClient, + Rooms: s.rooms, + Policy: s.federation.Policy(), + Logger: log, + }) + emit.add(queue) + s.fedModule = federationapi.NewModule(queue, log) + s.sync, err = syncapi.New(syncapi.Options{ ServerName: cfg.Global.ServerName, Rooms: s.rooms, @@ -191,10 +276,36 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) _ = s.closeDB() return nil, err } + s.sync.SetOutbound(federationapi.NewEDUs(queue, s.rooms)) + + s.fedAPI, err = federationapi.NewServer(federationapi.ServerOptions{ + ServerName: cfg.Global.ServerName, + Gate: s.federation, + Keys: s.keyRing, + Rooms: s.rooms, + Ephemeral: s.sync, + Profiles: s.users, + Store: s.db.Federation(), + Logger: log, + }) + if err != nil { + _ = s.closeDB() + return nil, err + } } else { log.Warn("started without a signing key: rooms and /sync are unavailable") } + // The interfaces below are left nil rather than set to a nil pointer + // when there is no federation: a nil *federation.Client in an + // interface is not a nil interface, and would be called. + var remoteProfiles clientapi.RemoteProfiles + var fedStatus adminapi.FederationStatus + if s.fedClient != nil { + remoteProfiles = s.fedClient + fedStatus = federationStatus{module: s.fedModule} + } + s.clientAPI, err = clientapi.New(clientapi.Options{ ServerName: cfg.Global.ServerName, ClientBaseURL: cfg.ClientBaseURL(), @@ -203,6 +314,7 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) Sync: s.sync, Settings: settingsService, Federation: s.federation, + RemoteProfiles: remoteProfiles, Registration: cfg.Registration, RateLimit: cfg.ClientAPI.RateLimit, TrustXForwardedFor: cfg.ClientAPI.TrustXForwardedFor, @@ -214,13 +326,14 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) } s.adminAPI, err = adminapi.New(adminapi.Options{ - ServerName: cfg.Global.ServerName, - Users: s.users, - Rooms: s.rooms, - Settings: settingsService, - Federation: s.federation, - Database: s.db, - Logger: log, + ServerName: cfg.Global.ServerName, + Users: s.users, + Rooms: s.rooms, + Settings: settingsService, + Federation: s.federation, + FederationStatus: fedStatus, + Database: s.db, + Logger: log, }) if err != nil { _ = s.closeDB() @@ -292,29 +405,65 @@ func (s *Server) SigningKey() *signing.KeyPair { return s.signingKey } // Run listens and serves until ctx is cancelled, then shuts down // gracefully: it stops accepting connections and gives in-flight requests // up to listen.shutdown_timeout to finish. +// +// With listen.federation_address set there are two listeners, the plain +// one and a TLS one for other servers, serving the same handler: what a +// request may reach is decided by the gate and the routes, not by which +// port it came in on. The federation module is started here rather than +// in New, so that a server that is built and never run leaves nothing +// running behind it, and is stopped before the listeners close. func (s *Server) Run(ctx context.Context) error { listener, err := net.Listen("tcp", s.cfg.Listen.Address) if err != nil { return fmt.Errorf("listen on %s: %w", s.cfg.Listen.Address, err) } + listeners := []net.Listener{listener} + if s.tlsConfig != nil { + fedListener, err := net.Listen("tcp", s.cfg.Listen.FederationAddress) + if err != nil { + _ = listener.Close() + return fmt.Errorf("listen on %s: %w", s.cfg.Listen.FederationAddress, err) + } + listeners = append(listeners, tls.NewListener(fedListener, s.tlsConfig)) + } addr := listener.Addr().String() s.addr.Store(&addr) defer s.addr.Store(nil) - httpServer := &http.Server{ - Handler: s.handler, - ReadHeaderTimeout: s.cfg.Listen.ReadHeaderTimeout, - ReadTimeout: s.cfg.Listen.ReadTimeout, - WriteTimeout: s.cfg.Listen.WriteTimeout, - IdleTimeout: s.cfg.Listen.IdleTimeout, - MaxHeaderBytes: s.cfg.Listen.MaxHeaderBytes, - ErrorLog: slog.NewLogLogger(s.log.Handler(), slog.LevelDebug), - BaseContext: func(net.Listener) context.Context { return ctx }, + newHTTPServer := func() *http.Server { + return &http.Server{ + Handler: s.handler, + ReadHeaderTimeout: s.cfg.Listen.ReadHeaderTimeout, + ReadTimeout: s.cfg.Listen.ReadTimeout, + WriteTimeout: s.cfg.Listen.WriteTimeout, + IdleTimeout: s.cfg.Listen.IdleTimeout, + MaxHeaderBytes: s.cfg.Listen.MaxHeaderBytes, + ErrorLog: slog.NewLogLogger(s.log.Handler(), slog.LevelDebug), + BaseContext: func(net.Listener) context.Context { return ctx }, + } } + s.running.Store(true) + defer s.running.Store(false) + if s.fedModule != nil { + if s.settings.FederationEnabled() { + if err := s.fedModule.Start(ctx); err != nil { + s.log.Error("could not start federation", "error", err) + } + } + defer s.fedModule.Stop() + } + + federationAddr := "" + if len(listeners) > 1 { + federationAddr = listeners[1].Addr().String() + s.fedAddr.Store(&federationAddr) + defer s.fedAddr.Store(nil) + } s.log.Info("warp is listening", "address", addr, + "federation_address", federationAddr, "server_name", s.cfg.Global.ServerName, "version", buildinfo.String(), "database", s.db.Driver(), @@ -323,32 +472,52 @@ func (s *Server) Run(ctx context.Context) error { "presence", s.cfg.Sync.Presence, ) - errCh := make(chan error, 1) - go func() { - err := httpServer.Serve(listener) - if errors.Is(err, http.ErrServerClosed) { - err = nil - } - errCh <- err - }() + servers := make([]*http.Server, len(listeners)) + errCh := make(chan error, len(listeners)) + for i, l := range listeners { + servers[i] = newHTTPServer() + go func(srv *http.Server, l net.Listener) { + err := srv.Serve(l) + if errors.Is(err, http.ErrServerClosed) { + err = nil + } + errCh <- err + }(servers[i], l) + } + var serveErr error select { - case err := <-errCh: - return err + case serveErr = <-errCh: + // One listener failing takes the server down: half a homeserver + // is worse than a restart. case <-ctx.Done(): } s.log.Info("shutting down", "timeout", s.cfg.Listen.ShutdownTimeout) + if s.fedModule != nil { + s.fedModule.Stop() + } shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.cfg.Listen.ShutdownTimeout) defer cancel() - if err := httpServer.Shutdown(shutdownCtx); err != nil { - s.log.Warn("graceful shutdown timed out; closing connections", "error", err) - _ = httpServer.Close() + for _, srv := range servers { + if err := srv.Shutdown(shutdownCtx); err != nil { + s.log.Warn("graceful shutdown timed out; closing connections", "error", err) + _ = srv.Close() + } } - // Serve has returned by now; surface anything it reported. - if err := <-errCh; err != nil { - return err + // Every Serve has returned by now; surface anything they reported. + remaining := len(servers) + if serveErr != nil { + remaining-- + } + for range remaining { + if err := <-errCh; err != nil && serveErr == nil { + serveErr = err + } + } + if serveErr != nil { + return serveErr } s.log.Info("stopped") return nil @@ -356,6 +525,9 @@ func (s *Server) Run(ctx context.Context) error { // Close releases the resources New acquired. func (s *Server) Close() error { + if s.fedModule != nil { + s.fedModule.Stop() + } if s.sync != nil { // The typing tracker holds a timer per typing user; a server // shutting down should not wait for them. @@ -378,3 +550,17 @@ func (s *Server) closeDB() error { func contextWithTimeout(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) { return context.WithTimeout(ctx, d) } + +// FederationModule exposes the federation module, or nil when the server +// started without a signing key. Tests that never call Run start it +// themselves. +func (s *Server) FederationModule() *federationapi.Module { return s.fedModule } + +// FederationAddr reports the address of the TLS federation listener while +// Run is serving one, or "". +func (s *Server) FederationAddr() string { + if p := s.fedAddr.Load(); p != nil { + return *p + } + return "" +} diff --git a/server/internal/storage/backfill_test.go b/server/internal/storage/backfill_test.go new file mode 100644 index 0000000..dbbe6ab --- /dev/null +++ b/server/internal/storage/backfill_test.go @@ -0,0 +1,205 @@ +package storage_test + +import ( + "slices" + "testing" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Backfilled history: events numbered downwards from -1, below everything +// the room already holds, and every stream read that has to find them +// there. + +// timelineIDs reads a window and returns its event IDs in the order the +// store gave them. +func timelineIDs(t *testing.T, rooms storage.RoomStore, req storage.TimelineRequest) []string { + t.Helper() + req.RoomID = testRoomID + if req.Limit == 0 { + req.Limit = 100 + } + events, err := rooms.Timeline(t.Context(), req) + if err != nil { + t.Fatalf("Timeline(%+v): %v", req, err) + } + return eventIDsOf(events) +} + +func TestBackfillSortsBelowEverythingAlreadyHeld(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + rooms := migratedRoomDB(t, driver).Rooms() + if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { + t.Fatalf("CreateRoom: %v", err) + } + + // The room as a server that joined it part-way through holds + // it: the state it was handed as outliers, then its own join + // and what followed in the timeline. + create := newTestEvent(testRoomID, "$create", "m.room.create", stateKey("")) + topic := newTestEvent(testRoomID, "$topic", "m.room.topic", stateKey("")) + if err := rooms.WriteOutliers(ctx, testRoomID, []storage.StoredEvent{create, topic}); err != nil { + t.Fatalf("WriteOutliers: %v", err) + } + state := []storage.StateEntry{ + {Type: "m.room.create", StateKey: "", EventID: "$create"}, + {Type: "m.room.topic", StateKey: "", EventID: "$topic"}, + } + if _, err := rooms.Write(ctx, storage.RoomWrite{ + Event: newTestEvent(testRoomID, "$join", "m.room.member", stateKey("@bob:warp.example")), + StateBefore: state, + StateDelta: append(slices.Clone(state), storage.StateEntry{ + Type: "m.room.member", StateKey: "@bob:warp.example", EventID: "$join", Membership: "join", + }), + StateDeltaIsWholeState: true, + Extremities: []string{"$join"}, + }); err != nil { + t.Fatalf("Write the join: %v", err) + } + writeMessages(t, rooms, testRoomID, "$live") + topBefore, err := rooms.MaxStreamOrdering(ctx) + if err != nil { + t.Fatalf("MaxStreamOrdering: %v", err) + } + + // The history before the join, oldest first. The create event + // and the topic are outliers already and are promoted; the + // message is new. + if err := rooms.WriteBackfill(ctx, testRoomID, []storage.BackfillEvent{ + {Event: create}, + {Event: topic, StateBefore: state[:1]}, + {Event: newTestEvent(testRoomID, "$old", "m.room.message", nil), StateBefore: state}, + }); err != nil { + t.Fatalf("WriteBackfill: %v", err) + } + + want := map[string]int64{"$create": -3, "$topic": -2, "$old": -1} + for id, pos := range want { + stored, err := rooms.Event(ctx, testRoomID, id) + if err != nil { + t.Fatalf("Event(%s): %v", id, err) + } + if stored.StreamOrdering != pos || stored.Outlier { + t.Errorf("%s: position %d, outlier %v; want %d in the timeline", + id, stored.StreamOrdering, stored.Outlier, pos) + } + } + + // The whole room, forwards and unbounded, is history first. + wantAll := []string{"$create", "$topic", "$old", "$join", "$live"} + if got := timelineIDs(t, rooms, storage.TimelineRequest{Direction: storage.DirectionForward}); !slices.Equal(got, wantAll) { + t.Errorf("forwards from the start = %v, want %v", got, wantAll) + } + + // Negative bounds, both ends, both directions. + for _, tc := range []struct { + name string + req storage.TimelineRequest + want []string + }{ + { + name: "backwards from below the first live event", + req: storage.TimelineRequest{High: storage.PositionBefore(1), Direction: storage.DirectionBackward}, + want: []string{"$old", "$topic", "$create"}, + }, + { + name: "backwards from a negative position", + req: storage.TimelineRequest{High: -2, Direction: storage.DirectionBackward}, + want: []string{"$topic", "$create"}, + }, + { + name: "forwards past a negative position into the live stream", + req: storage.TimelineRequest{Low: -2, Direction: storage.DirectionForward, Limit: 2}, + want: []string{"$old", "$join"}, + }, + { + name: "a window entirely below zero", + req: storage.TimelineRequest{Low: -3, High: -1, Direction: storage.DirectionForward}, + want: []string{"$topic", "$old"}, + }, + } { + if got := timelineIDs(t, rooms, tc.req); !slices.Equal(got, tc.want) { + t.Errorf("%s: %v, want %v", tc.name, got, tc.want) + } + } + + // History changes neither the top of the stream nor anything + // an incremental sync looks at. + topAfter, err := rooms.MaxStreamOrdering(ctx) + if err != nil { + t.Fatalf("MaxStreamOrdering: %v", err) + } + if topAfter > topBefore { + t.Errorf("the stream top moved from %d to %d", topBefore, topAfter) + } + if changed, err := rooms.RoomsWithEvents(ctx, []string{testRoomID}, topAfter, 0); err != nil { + t.Fatalf("RoomsWithEvents: %v", err) + } else if len(changed) != 0 { + t.Errorf("RoomsWithEvents since the top = %v, want nothing", changed) + } + if changed, err := rooms.RoomsWithEvents(ctx, []string{testRoomID}, 0, -1); err != nil { + t.Fatalf("RoomsWithEvents: %v", err) + } else if len(changed) != 1 { + t.Errorf("RoomsWithEvents over the history = %v, want the room", changed) + } + extremities, err := rooms.Extremities(ctx, testRoomID) + if err != nil { + t.Fatalf("Extremities: %v", err) + } + if len(extremities) != 1 || extremities[0].EventID != "$live" { + t.Errorf("extremities = %v, want just $live", eventIDsOf(extremities)) + } + current, err := rooms.CurrentState(ctx, testRoomID) + if err != nil { + t.Fatalf("CurrentState: %v", err) + } + if len(current) != 3 { + t.Errorf("current state = %+v, want the three tuples from before", current) + } + + // The state at a position inside the history is the snapshot + // recorded against the event there, plus the event. + for _, tc := range []struct { + pos int64 + want int + }{{-4, 0}, {-3, 1}, {-2, 2}, {-1, 2}} { + entries, err := rooms.StateAtStream(ctx, testRoomID, tc.pos) + if err != nil { + t.Fatalf("StateAtStream(%d): %v", tc.pos, err) + } + if len(entries) != tc.want { + t.Errorf("StateAtStream(%d) = %+v, want %d entries", tc.pos, entries, tc.want) + } + } + + // A second batch goes below the first. An event already in the + // timeline is skipped and takes no position. + if err := rooms.WriteBackfill(ctx, testRoomID, []storage.BackfillEvent{ + {Event: newTestEvent(testRoomID, "$older", "m.room.message", nil), StateBefore: state}, + {Event: newTestEvent(testRoomID, "$old", "m.room.message", nil), StateBefore: state}, + }); err != nil { + t.Fatalf("WriteBackfill again: %v", err) + } + older, err := rooms.Event(ctx, testRoomID, "$older") + if err != nil { + t.Fatalf("Event($older): %v", err) + } + if older.StreamOrdering != -4 { + t.Errorf("$older is at %d, want -4", older.StreamOrdering) + } + if old, _ := rooms.Event(ctx, testRoomID, "$old"); old.StreamOrdering != -1 { + t.Errorf("$old moved to %d", old.StreamOrdering) + } + }) + } +} + +func TestPositionBeforeSkipsZero(t *testing.T) { + for pos, want := range map[int64]int64{5: 4, 2: 1, 1: -1, -1: -2, -7: -8} { + if got := storage.PositionBefore(pos); got != want { + t.Errorf("PositionBefore(%d) = %d, want %d", pos, got, want) + } + } +} diff --git a/server/internal/storage/federation.go b/server/internal/storage/federation.go new file mode 100644 index 0000000..64e448f --- /dev/null +++ b/server/internal/storage/federation.go @@ -0,0 +1,123 @@ +package storage + +import "context" + +// ServerKey is one signing key of another server, as its key server or a +// notary reported it. +type ServerKey struct { + ServerName string + + // KeyID is the full identifier, algorithm included: "ed25519:abc". + KeyID string + + // PublicKey is the unpadded base64 key, as it is on the wire. + PublicKey string + + // ValidUntilTS is how long the server vouched for the key, in + // milliseconds since the epoch. + ValidUntilTS int64 + + // ExpiredTS is when the server stopped signing with the key, for one + // it listed as an old_verify_key; zero for a key still in use. + ExpiredTS int64 + + FetchedAtMS int64 +} + +// The kinds of item in the outbound queue. +const ( + QueueKindPDU = "pdu" + QueueKindEDU = "edu" +) + +// QueuedItem is one PDU or EDU waiting for a destination. +type QueuedItem struct { + // ID is assigned by Enqueue and orders a destination's items. + ID int64 + + Destination string + + // Kind is QueueKindPDU or QueueKindEDU. + Kind string + + // RoomID and EventID describe a PDU, for logs and the admin API. + RoomID string + EventID string + + // Payload is the JSON that goes on the wire. + Payload []byte + + QueuedAtMS int64 +} + +// Destination is the delivery health of one remote server. +type Destination struct { + ServerName string + + // Failures is how many sends in a row have failed. Zero is a healthy + // destination. + Failures int + + // RetryAtMS is when the next attempt may be made; zero means now. + RetryAtMS int64 + + LastSuccessMS int64 + LastFailureMS int64 + LastError string +} + +// DestinationSummary is a destination with the size of its queue, for the +// admin API. +type DestinationSummary struct { + Destination + + QueuedPDUs int + QueuedEDUs int +} + +// FederationStore persists what federation needs to survive a restart: +// other servers' keys, the outbound queue, the per-destination backoff and +// the log of inbound transactions. +type FederationStore interface { + // StoreServerKeys inserts or replaces keys. A key already stored is + // overwritten with the newer report of it. + StoreServerKeys(ctx context.Context, keys []ServerKey) error + + // ServerKeys returns the stored keys of a server. An empty keyIDs + // returns all of them. + ServerKeys(ctx context.Context, serverName string, keyIDs []string) ([]ServerKey, error) + + // Enqueue adds items to the outbound queue, assigning their IDs. + Enqueue(ctx context.Context, items []QueuedItem) ([]QueuedItem, error) + + // Queued returns up to limit of a destination's items of one kind, + // oldest first. + Queued(ctx context.Context, destination, kind string, limit int) ([]QueuedItem, error) + + // Dequeue removes delivered items. + Dequeue(ctx context.Context, ids []int64) error + + // QueuedDestinations lists the destinations that have anything queued, + // which is where a restarted queue picks up. + QueuedDestinations(ctx context.Context) ([]string, error) + + // Destination reads one destination's delivery health. A server that + // has never been sent anything comes back as a zero Destination with + // its name filled in and no error. + Destination(ctx context.Context, serverName string) (Destination, error) + + // SetDestination records a destination's delivery health. + SetDestination(ctx context.Context, d Destination) error + + // Destinations lists every destination known to the queue, with its + // queue depth, ordered by server name. + Destinations(ctx context.Context) ([]DestinationSummary, error) + + // InboundTransaction returns the response recorded for a transaction + // already received from origin, and whether there was one. + InboundTransaction(ctx context.Context, origin, txnID string) ([]byte, bool, error) + + // RecordInboundTransaction stores the response to a transaction so + // that a retransmission is answered the same way. + RecordInboundTransaction(ctx context.Context, origin, txnID string, response []byte, atMS int64) error +} diff --git a/server/internal/storage/federation_test.go b/server/internal/storage/federation_test.go new file mode 100644 index 0000000..f79f3a1 --- /dev/null +++ b/server/internal/storage/federation_test.go @@ -0,0 +1,431 @@ +package storage_test + +import ( + "database/sql" + "os" + "slices" + "testing" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// migratedFederationDB opens a driver with the room and federation tables +// empty. +func migratedFederationDB(t *testing.T, driver string) storage.Database { + t.Helper() + db := migratedRoomDB(t, driver) + if driver == storage.DriverPostgres { + pool, err := sql.Open("pgx", os.Getenv("WARP_TEST_POSTGRES_DSN")) + if err != nil { + t.Fatalf("open postgres to reset the federation tables: %v", err) + } + defer func() { _ = pool.Close() }() + if _, err := pool.ExecContext(t.Context(), `TRUNCATE + federation_server_keys, federation_queue, + federation_destinations, federation_inbound_txns`); err != nil { + t.Fatalf("reset the federation tables: %v", err) + } + if _, err := pool.ExecContext(t.Context(), `UPDATE federation_sequences SET value = 0`); err != nil { + t.Fatalf("rewind the federation counters: %v", err) + } + } + return db +} + +func TestOutliersAreReadableButNotInTheTimeline(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + rooms := migratedFederationDB(t, driver).Rooms() + if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { + t.Fatalf("CreateRoom: %v", err) + } + + outliers := []storage.StoredEvent{ + newTestEvent(testRoomID, "$create", "m.room.create", stateKey("")), + newTestEvent(testRoomID, "$member", "m.room.member", stateKey("@bob:remote.example")), + } + if err := rooms.WriteOutliers(ctx, testRoomID, outliers); err != nil { + t.Fatalf("WriteOutliers: %v", err) + } + // Writing them again is not an error: an event ID names its + // contents, so the second copy is the first one. + if err := rooms.WriteOutliers(ctx, testRoomID, outliers); err != nil { + t.Fatalf("WriteOutliers twice: %v", err) + } + + got, err := rooms.EventsByID(ctx, testRoomID, []string{"$create", "$member"}) + if err != nil { + t.Fatalf("EventsByID: %v", err) + } + if len(got) != 2 { + t.Fatalf("EventsByID found %d outliers, want 2", len(got)) + } + for _, e := range got { + if !e.Outlier { + t.Errorf("%s came back as a timeline event", e.EventID) + } + } + + // A join that cites the outliers as its state. + join := newTestEvent(testRoomID, "$join", "m.room.member", stateKey("@alice:warp.example")) + if _, err := rooms.Write(ctx, storage.RoomWrite{ + Event: join, + StateBefore: []storage.StateEntry{ + {Type: "m.room.create", EventID: "$create"}, + {Type: "m.room.member", StateKey: "@bob:remote.example", EventID: "$member", Membership: "join"}, + }, + StateDelta: []storage.StateEntry{ + {Type: "m.room.create", EventID: "$create"}, + {Type: "m.room.member", StateKey: "@bob:remote.example", EventID: "$member", Membership: "join"}, + {Type: "m.room.member", StateKey: "@alice:warp.example", EventID: "$join", Membership: "join"}, + }, + StateDeltaIsWholeState: true, + Extremities: []string{"$join"}, + }); err != nil { + t.Fatalf("Write join: %v", err) + } + + timeline, err := rooms.Timeline(ctx, storage.TimelineRequest{ + RoomID: testRoomID, Limit: 10, Direction: storage.DirectionBackward, + }) + if err != nil { + t.Fatalf("Timeline: %v", err) + } + if ids := eventIDsOf(timeline); !slices.Equal(ids, []string{"$join"}) { + t.Errorf("timeline = %v, want only the join", ids) + } + + // The current state reads through to the outliers: that is + // the whole point of keeping them in the events table. + state, err := rooms.CurrentStateEvents(ctx, testRoomID) + if err != nil { + t.Fatalf("CurrentStateEvents: %v", err) + } + if len(state) != 3 { + t.Errorf("current state has %d events, want 3", len(state)) + } + + top, err := rooms.MaxStreamOrdering(ctx) + if err != nil { + t.Fatalf("MaxStreamOrdering: %v", err) + } + at, err := rooms.StateAtStream(ctx, testRoomID, top) + if err != nil { + t.Fatalf("StateAtStream: %v", err) + } + if len(at) != 3 { + t.Errorf("state at the join has %d entries, want 3", len(at)) + } + }) + } +} + +func TestOutlierIsPromotedWhenItArrivesAgainAsAnEvent(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + rooms := migratedFederationDB(t, driver).Rooms() + if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { + t.Fatalf("CreateRoom: %v", err) + } + writeMessages(t, rooms, testRoomID, "$first") + if err := rooms.WriteOutliers(ctx, testRoomID, []storage.StoredEvent{ + newTestEvent(testRoomID, "$late", "m.room.message", nil), + }); err != nil { + t.Fatalf("WriteOutliers: %v", err) + } + writeMessages(t, rooms, testRoomID, "$late") + + stored, err := rooms.Event(ctx, testRoomID, "$late") + if err != nil { + t.Fatalf("Event: %v", err) + } + if stored.Outlier { + t.Error("the event is still an outlier after being written as one") + } + timeline, err := rooms.Timeline(ctx, storage.TimelineRequest{ + RoomID: testRoomID, Limit: 10, Direction: storage.DirectionForward, + }) + if err != nil { + t.Fatalf("Timeline: %v", err) + } + if ids := eventIDsOf(timeline); !slices.Equal(ids, []string{"$first", "$late"}) { + t.Errorf("timeline = %v, want [$first $late]", ids) + } + }) + } +} + +func TestInviteStateLastsUntilTheNextMembershipChange(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + rooms := migratedFederationDB(t, driver).Rooms() + if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { + t.Fatalf("CreateRoom: %v", err) + } + const invitee = "@alice:warp.example" + stripped := []byte(`[{"type":"m.room.name","state_key":"","sender":"@bob:remote.example","content":{"name":"Hi"}}]`) + + invite := newTestEvent(testRoomID, "$invite", "m.room.member", stateKey(invitee)) + if _, err := rooms.Write(ctx, storage.RoomWrite{ + Event: invite, + Outlier: true, + StateDelta: []storage.StateEntry{{Type: "m.room.member", StateKey: invitee, EventID: "$invite", Membership: "invite"}}, + InviteState: stripped, + }); err != nil { + t.Fatalf("Write invite: %v", err) + } + + got, err := rooms.InviteState(ctx, testRoomID, invitee) + if err != nil { + t.Fatalf("InviteState: %v", err) + } + if string(got) != string(stripped) { + t.Errorf("InviteState = %s, want %s", got, stripped) + } + memberships, err := rooms.MembershipsForUser(ctx, invitee, []string{"invite"}) + if err != nil { + t.Fatalf("MembershipsForUser: %v", err) + } + if len(memberships) != 1 || memberships[0].StreamOrdering == 0 { + t.Errorf("an outlier invite must still have a stream position for /sync, got %+v", memberships) + } + + leave := newTestEvent(testRoomID, "$leave", "m.room.member", stateKey(invitee)) + if _, err := rooms.Write(ctx, storage.RoomWrite{ + Event: leave, + Outlier: true, + StateDelta: []storage.StateEntry{{Type: "m.room.member", StateKey: invitee, EventID: "$leave", Membership: "leave"}}, + }); err != nil { + t.Fatalf("Write leave: %v", err) + } + if got, err := rooms.InviteState(ctx, testRoomID, invitee); err != nil || got != nil { + t.Errorf("InviteState after a rejection = %s, %v; want nothing", got, err) + } + }) + } +} + +func TestServerKeysAreUpsertedAndFiltered(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + fed := migratedFederationDB(t, driver).Federation() + + if err := fed.StoreServerKeys(ctx, []storage.ServerKey{ + {ServerName: "remote.example", KeyID: "ed25519:a", PublicKey: "AAA", ValidUntilTS: 10, FetchedAtMS: 1}, + {ServerName: "remote.example", KeyID: "ed25519:b", PublicKey: "BBB", ValidUntilTS: 20, ExpiredTS: 15, FetchedAtMS: 1}, + {ServerName: "other.example", KeyID: "ed25519:a", PublicKey: "CCC", ValidUntilTS: 30, FetchedAtMS: 1}, + }); err != nil { + t.Fatalf("StoreServerKeys: %v", err) + } + if err := fed.StoreServerKeys(ctx, []storage.ServerKey{ + {ServerName: "remote.example", KeyID: "ed25519:a", PublicKey: "AAA", ValidUntilTS: 99, FetchedAtMS: 2}, + }); err != nil { + t.Fatalf("StoreServerKeys again: %v", err) + } + + all, err := fed.ServerKeys(ctx, "remote.example", nil) + if err != nil { + t.Fatalf("ServerKeys: %v", err) + } + if len(all) != 2 { + t.Fatalf("ServerKeys found %d keys, want 2", len(all)) + } + if all[0].KeyID != "ed25519:a" || all[0].ValidUntilTS != 99 { + t.Errorf("the refetched key was not replaced: %+v", all[0]) + } + if all[1].ExpiredTS != 15 { + t.Errorf("an old key lost its expiry: %+v", all[1]) + } + + one, err := fed.ServerKeys(ctx, "remote.example", []string{"ed25519:b"}) + if err != nil { + t.Fatalf("ServerKeys(b): %v", err) + } + if len(one) != 1 || one[0].PublicKey != "BBB" { + t.Errorf("ServerKeys(b) = %+v", one) + } + }) + } +} + +func TestQueueIsPerDestinationAndInOrder(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + fed := migratedFederationDB(t, driver).Federation() + + queued, err := fed.Enqueue(ctx, []storage.QueuedItem{ + {Destination: "a.example", Kind: storage.QueueKindPDU, EventID: "$1", Payload: []byte(`{"n":1}`)}, + {Destination: "b.example", Kind: storage.QueueKindPDU, EventID: "$1", Payload: []byte(`{"n":1}`)}, + {Destination: "a.example", Kind: storage.QueueKindEDU, Payload: []byte(`{"edu":true}`)}, + {Destination: "a.example", Kind: storage.QueueKindPDU, EventID: "$2", Payload: []byte(`{"n":2}`)}, + }) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + for i := 1; i < len(queued); i++ { + if queued[i].ID <= queued[i-1].ID { + t.Fatalf("queue IDs are not increasing: %d then %d", queued[i-1].ID, queued[i].ID) + } + } + + pdus, err := fed.Queued(ctx, "a.example", storage.QueueKindPDU, 10) + if err != nil { + t.Fatalf("Queued: %v", err) + } + if len(pdus) != 2 || pdus[0].EventID != "$1" || pdus[1].EventID != "$2" { + t.Fatalf("Queued(a, pdu) = %+v", pdus) + } + if string(pdus[1].Payload) != `{"n":2}` { + t.Errorf("payload = %s", pdus[1].Payload) + } + + destinations, err := fed.QueuedDestinations(ctx) + if err != nil { + t.Fatalf("QueuedDestinations: %v", err) + } + if !slices.Equal(destinations, []string{"a.example", "b.example"}) { + t.Errorf("QueuedDestinations = %v", destinations) + } + + if err := fed.Dequeue(ctx, []int64{pdus[0].ID}); err != nil { + t.Fatalf("Dequeue: %v", err) + } + left, err := fed.Queued(ctx, "a.example", storage.QueueKindPDU, 10) + if err != nil { + t.Fatalf("Queued after Dequeue: %v", err) + } + if len(left) != 1 || left[0].EventID != "$2" { + t.Errorf("after dequeue = %+v", left) + } + }) + } +} + +func TestDestinationHealth(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + fed := migratedFederationDB(t, driver).Federation() + + d, err := fed.Destination(ctx, "never.example") + if err != nil { + t.Fatalf("Destination: %v", err) + } + if d.ServerName != "never.example" || d.Failures != 0 { + t.Errorf("an unknown destination = %+v, want a healthy zero", d) + } + + if err := fed.SetDestination(ctx, storage.Destination{ + ServerName: "down.example", Failures: 3, RetryAtMS: 500, LastError: "connection refused", + }); err != nil { + t.Fatalf("SetDestination: %v", err) + } + if err := fed.SetDestination(ctx, storage.Destination{ + ServerName: "down.example", Failures: 4, RetryAtMS: 900, LastError: "timeout", + }); err != nil { + t.Fatalf("SetDestination again: %v", err) + } + if _, err := fed.Enqueue(ctx, []storage.QueuedItem{ + {Destination: "busy.example", Kind: storage.QueueKindPDU, Payload: []byte(`{}`)}, + {Destination: "busy.example", Kind: storage.QueueKindEDU, Payload: []byte(`{}`)}, + {Destination: "down.example", Kind: storage.QueueKindPDU, Payload: []byte(`{}`)}, + }); err != nil { + t.Fatalf("Enqueue: %v", err) + } + + all, err := fed.Destinations(ctx) + if err != nil { + t.Fatalf("Destinations: %v", err) + } + if len(all) != 2 { + t.Fatalf("Destinations = %+v, want busy and down", all) + } + busy, down := all[0], all[1] + if busy.ServerName != "busy.example" || busy.QueuedPDUs != 1 || busy.QueuedEDUs != 1 || busy.Failures != 0 { + t.Errorf("busy = %+v", busy) + } + if down.ServerName != "down.example" || down.Failures != 4 || down.LastError != "timeout" || down.QueuedPDUs != 1 { + t.Errorf("down = %+v", down) + } + }) + } +} + +func TestInboundTransactionsAreRememberedOnce(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + fed := migratedFederationDB(t, driver).Federation() + + if _, ok, err := fed.InboundTransaction(ctx, "remote.example", "t1"); err != nil || ok { + t.Fatalf("InboundTransaction before = %v, %v", ok, err) + } + if err := fed.RecordInboundTransaction(ctx, "remote.example", "t1", []byte(`{"pdus":{}}`), 1); err != nil { + t.Fatalf("RecordInboundTransaction: %v", err) + } + if err := fed.RecordInboundTransaction(ctx, "remote.example", "t1", []byte(`{"other":1}`), 2); err != nil { + t.Fatalf("RecordInboundTransaction again: %v", err) + } + got, ok, err := fed.InboundTransaction(ctx, "remote.example", "t1") + if err != nil || !ok || string(got) != `{"pdus":{}}` { + t.Errorf("InboundTransaction = %s, %v, %v; want the first response", got, ok, err) + } + }) + } +} + +func eventIDsOf(events []storage.StoredEvent) []string { + out := make([]string, 0, len(events)) + for _, e := range events { + out = append(out, e.EventID) + } + return out +} + +func TestHasJoinedMemberOn(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + rooms := migratedFederationDB(t, driver).Rooms() + if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { + t.Fatal(err) + } + for _, m := range []struct{ user, membership string }{ + {"@a:one.example", "join"}, + {"@b:two.example", "leave"}, + {"@c:x_y.example", "join"}, + } { + e := newTestEvent(testRoomID, "$"+m.user, "m.room.member", stateKey(m.user)) + if _, err := rooms.Write(ctx, storage.RoomWrite{ + Event: e, + StateDelta: []storage.StateEntry{{Type: "m.room.member", StateKey: m.user, EventID: e.EventID, Membership: m.membership}}, + Extremities: []string{e.EventID}, + }); err != nil { + t.Fatal(err) + } + } + cases := map[string]bool{ + "one.example": true, + "two.example": false, // left + "x_y.example": true, + "xay.example": false, // '_' is not a LIKE wildcard here + "e.example": false, // a suffix of a name is not the name + "none.example": false, + } + for server, want := range cases { + got, err := rooms.HasJoinedMemberOn(ctx, testRoomID, server) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("HasJoinedMemberOn(%s) = %v, want %v", server, got, want) + } + } + }) + } +} diff --git a/server/internal/storage/pocketbase/pocketbase.go b/server/internal/storage/pocketbase/pocketbase.go index e2251e9..56c0a5d 100644 --- a/server/internal/storage/pocketbase/pocketbase.go +++ b/server/internal/storage/pocketbase/pocketbase.go @@ -60,6 +60,9 @@ func (*Database) Rooms() storage.RoomStore { return notImplementedRooms{} } // Sync implements storage.Database. func (*Database) Sync() storage.SyncStore { return notImplementedSync{} } +// Federation implements storage.Database. +func (*Database) Federation() storage.FederationStore { return notImplementedFederation{} } + // Close implements storage.Database. func (*Database) Close() error { return nil } @@ -298,6 +301,75 @@ func (notImplementedRooms) RoomStats(context.Context) (storage.RoomStats, error) return storage.RoomStats{}, storage.ErrNotImplemented } +func (notImplementedRooms) WriteOutliers(context.Context, string, []storage.StoredEvent) error { + return storage.ErrNotImplemented +} + +func (notImplementedRooms) WriteBackfill(context.Context, string, []storage.BackfillEvent) error { + return storage.ErrNotImplemented +} + +func (notImplementedRooms) InviteState(context.Context, string, string) ([]byte, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedRooms) HasJoinedMemberOn(context.Context, string, string) (bool, error) { + return false, storage.ErrNotImplemented +} + +func (notImplementedRooms) EventByID(context.Context, string) (storage.StoredEvent, error) { + return storage.StoredEvent{}, storage.ErrNotImplemented +} + +// notImplementedFederation satisfies storage.FederationStore the same way. +// A queue that accepted events and never stored them would lose them the +// moment federation was switched on. +type notImplementedFederation struct{} + +func (notImplementedFederation) StoreServerKeys(context.Context, []storage.ServerKey) error { + return storage.ErrNotImplemented +} + +func (notImplementedFederation) ServerKeys(context.Context, string, []string) ([]storage.ServerKey, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedFederation) Enqueue(context.Context, []storage.QueuedItem) ([]storage.QueuedItem, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedFederation) Queued(context.Context, string, string, int) ([]storage.QueuedItem, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedFederation) Dequeue(context.Context, []int64) error { + return storage.ErrNotImplemented +} + +func (notImplementedFederation) QueuedDestinations(context.Context) ([]string, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedFederation) Destination(context.Context, string) (storage.Destination, error) { + return storage.Destination{}, storage.ErrNotImplemented +} + +func (notImplementedFederation) SetDestination(context.Context, storage.Destination) error { + return storage.ErrNotImplemented +} + +func (notImplementedFederation) Destinations(context.Context) ([]storage.DestinationSummary, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedFederation) InboundTransaction(context.Context, string, string) ([]byte, bool, error) { + return nil, false, storage.ErrNotImplemented +} + +func (notImplementedFederation) RecordInboundTransaction(context.Context, string, string, []byte, int64) error { + return storage.ErrNotImplemented +} + // notImplementedSync satisfies storage.SyncStore the same way. A sync that // returned an empty success would be worse than an error: a client would // take it for "nothing has happened" and stop asking. @@ -361,8 +433,9 @@ func (notImplementedSync) Filter(context.Context, string, string) ([]byte, error // Compile-time proof that the stub keeps up with the interfaces. var ( - _ storage.Database = (*Database)(nil) - _ storage.UserStore = notImplementedUsers{} - _ storage.RoomStore = notImplementedRooms{} - _ storage.SyncStore = notImplementedSync{} + _ storage.Database = (*Database)(nil) + _ storage.UserStore = notImplementedUsers{} + _ storage.RoomStore = notImplementedRooms{} + _ storage.SyncStore = notImplementedSync{} + _ storage.FederationStore = notImplementedFederation{} ) diff --git a/server/internal/storage/postgres/migrations/0005_federation.sql b/server/internal/storage/postgres/migrations/0005_federation.sql new file mode 100644 index 0000000..73b7fb1 --- /dev/null +++ b/server/internal/storage/postgres/migrations/0005_federation.sql @@ -0,0 +1,113 @@ +-- Federation: other servers' keys, the outbound queue and its per-server +-- backoff, the inbound transaction log, and the two additions the room +-- tables need once a room's events can come from somewhere else. +-- +-- Timestamps are milliseconds since the Unix epoch, as everywhere else in +-- Warp. + +-- An outlier is an event this server holds without holding the room's +-- state at it: the state and auth chain a join is handed by the resident +-- server, an auth event fetched to judge something else, an invite to a +-- room nobody here is in, an event that soft-failed. It is a real event — +-- it is signed, it was authorised against its own auth events, and other +-- events can cite it — but it has no place in the timeline: no state +-- snapshot (state_before_id stays 0), never a forward extremity, never +-- returned by /sync or /messages. +-- +-- It goes on the events table rather than into a table of its own because +-- every read of a room's current state joins the events table, and the +-- current state of a room joined over federation is made almost entirely +-- of outliers. An outlier that later arrives as an ordinary event — fetched +-- by /get_missing_events, say — is promoted in place. +ALTER TABLE roomserver_events ADD COLUMN outlier BOOLEAN NOT NULL DEFAULT FALSE; + +-- The stripped state an invite over federation arrives with. +-- +-- An invite to a room this server is not in is the invite event alone: the +-- room's state is on the inviting server. What a client shows in its +-- invite list — the room's name, its avatar, who invited you — comes from +-- the invite_room_state the inviting server sent along, which is stored +-- here until the invite is answered and then deleted with it. +CREATE TABLE IF NOT EXISTS roomserver_invite_state ( + room_id TEXT NOT NULL REFERENCES roomserver_rooms (room_id) ON DELETE CASCADE, + user_id TEXT NOT NULL, + -- A JSON array of stripped state events, exactly as received. + state_json TEXT NOT NULL, + PRIMARY KEY (room_id, user_id) +); + +-- Other servers' signing keys, as their key servers or a notary reported +-- them. A key is never deleted: an event signed years ago is still +-- verified with the key that signed it, and a server that has rotated its +-- key lists the old one only as an old_verify_key with the time it expired. +CREATE TABLE IF NOT EXISTS federation_server_keys ( + server_name TEXT NOT NULL, + key_id TEXT NOT NULL, + -- Unpadded base64, as it is on the wire. + public_key TEXT NOT NULL, + -- How long the server vouched for the key when it was fetched. A key + -- is re-fetched once this passes, and an event from room version 5 on + -- may not be verified with a key whose validity ended before the event + -- was sent. + valid_until_ts BIGINT NOT NULL, + -- When the key stopped being used, for an old_verify_key; 0 for a key + -- the server still signs with. + expired_ts BIGINT NOT NULL DEFAULT 0, + fetched_at_ms BIGINT NOT NULL, + PRIMARY KEY (server_name, key_id) +); + +-- One counter row per stream; see roomserver_sequences for why these are +-- rows and not database sequences. +CREATE TABLE IF NOT EXISTS federation_sequences ( + name TEXT NOT NULL PRIMARY KEY, + value BIGINT NOT NULL +); + +INSERT INTO federation_sequences (name, value) VALUES ('queue', 0) + ON CONFLICT (name) DO NOTHING; + +-- The outbound queue: every PDU and EDU waiting for a destination. +-- +-- A row is deleted once the destination has acknowledged the transaction +-- that carried it, so what is here after a restart is exactly what still +-- has to be sent. The payload is the JSON that goes on the wire, copied at +-- enqueue time: re-reading a PDU from the room tables at send time would +-- send its redacted form if it was redacted in the meantime, and that is +-- not the event the destination has to verify. +CREATE TABLE IF NOT EXISTS federation_queue ( + queue_id BIGINT NOT NULL PRIMARY KEY, + destination TEXT NOT NULL, + -- 'pdu' or 'edu'. + kind TEXT NOT NULL, + room_id TEXT NOT NULL DEFAULT '', + event_id TEXT NOT NULL DEFAULT '', + payload TEXT NOT NULL, + queued_at_ms BIGINT NOT NULL +); + +CREATE INDEX IF NOT EXISTS federation_queue_destination_idx + ON federation_queue (destination, kind, queue_id); + +-- Per-destination delivery health: how often sending has failed in a row +-- and when it may next be tried. A destination with no row has never +-- failed. +CREATE TABLE IF NOT EXISTS federation_destinations ( + server_name TEXT NOT NULL PRIMARY KEY, + failures INTEGER NOT NULL DEFAULT 0, + retry_at_ms BIGINT NOT NULL DEFAULT 0, + last_success_ms BIGINT NOT NULL DEFAULT 0, + last_failure_ms BIGINT NOT NULL DEFAULT 0, + last_error TEXT NOT NULL DEFAULT '' +); + +-- Transactions received from other servers, so that a retransmission is +-- answered with the response the first delivery got instead of being +-- processed a second time. +CREATE TABLE IF NOT EXISTS federation_inbound_txns ( + origin TEXT NOT NULL, + txn_id TEXT NOT NULL, + response_json TEXT NOT NULL, + received_at_ms BIGINT NOT NULL, + PRIMARY KEY (origin, txn_id) +); diff --git a/server/internal/storage/postgres/migrations/0006_backfill.sql b/server/internal/storage/postgres/migrations/0006_backfill.sql new file mode 100644 index 0000000..b33e307 --- /dev/null +++ b/server/internal/storage/postgres/migrations/0006_backfill.sql @@ -0,0 +1,15 @@ +-- Outbound backfill: history fetched from other servers for a room this +-- server joined part-way through. +-- +-- roomserver_events.stream_ordering is the order clients are shown a room +-- in, and the stream_ordering counter only ever goes up: every event it +-- numbers arrived after the ones before it. Backfilled history arrives +-- after everything too, but belongs before all of it. So it is numbered by +-- a second counter that runs the other way — from -1 downwards — and every +-- backfilled event sits below every event this server held when it came. +-- +-- Zero is neither counter's: the stream_ordering counter's first value is +-- 1 and this one's is -1, and a window bound of zero keeps meaning +-- "unbounded at that end". +INSERT INTO roomserver_sequences (name, value) VALUES ('backfill_ordering', 0) + ON CONFLICT (name) DO NOTHING; diff --git a/server/internal/storage/rooms.go b/server/internal/storage/rooms.go index bb2872b..82d8723 100644 --- a/server/internal/storage/rooms.go +++ b/server/internal/storage/rooms.go @@ -120,9 +120,12 @@ type StoredEvent struct { // JSON is the event as canonical JSON, exactly as it was signed. JSON []byte - // StreamOrdering is this server's own monotonic position for the event: - // the order it was persisted in, which is the order /sync and /messages - // hand events to clients. + // StreamOrdering is this server's own position for the event: the + // order /sync and /messages hand events to clients. For an event that + // arrived live it is the order it was persisted in, counting up from 1; + // for history backfilled from another server it is negative, counting + // down from -1, so that it sorts below everything that was already + // here. It is never zero. StreamOrdering int64 // RedactedBy is the event ID of the redaction that applies to this @@ -130,6 +133,13 @@ type StoredEvent struct { // applied, so a redacted event is already stripped on the way out; this // field is what lets unsigned.redacted_because be filled in. RedactedBy string + + // Outlier reports an event held without the room's state at it: part + // of a join's state or auth chain, an auth event fetched to judge + // another, an invite to a room nobody here is in, or an event that + // soft-failed. An outlier can be cited and read by ID, and is never in + // a timeline. + Outlier bool } // IsState reports whether the event is a state event. @@ -193,6 +203,32 @@ type RoomWrite struct { // replaces the stored JSON. Redacts string RedactedJSON []byte + + // Outlier writes the event as an outlier: no state snapshot is + // recorded for it and StateBefore is ignored. StateDelta and + // Extremities still apply when they are set, which is how an invite to + // a room nobody here is in becomes the one piece of that room's state + // this server knows. + Outlier bool + + // InviteState is the stripped state an invite over federation arrived + // with, kept for the invitee until the invite is answered. Any + // m.room.member write clears what was stored for its state key first, + // so the invite's own write is the only one that leaves a row behind. + InviteState []byte +} + +// BackfillEvent is one event of a room's history, fetched from another +// server after this server already held later events, with the room state +// it was authorised against. +type BackfillEvent struct { + Event StoredEvent + + // StateBefore is the state of the room as of the event's prev_events, + // recorded as its state snapshot exactly as RoomWrite.StateBefore is. + // It is what lets a client paging back through backfilled history be + // told the state at any point of it. + StateBefore []StateEntry } // RoomAlias is a local alias and the room it points at. @@ -342,7 +378,9 @@ type RoomStore interface { RoomsWithEvents(ctx context.Context, roomIDs []string, from, to int64) ([]string, error) // StateAtStream returns the room's state as of a stream position: the - // state after the last event at or before pos. + // state after the last event at or before pos. The position may be a + // negative one inside backfilled history; zero has no events at or + // before it that mean anything and returns nothing. // // It answers the two questions current state cannot. A user who has // left sees the room as it was when they left, and a sync with a gap in @@ -393,4 +431,39 @@ type RoomStore interface { // RoomStats returns the aggregate counts for the admin dashboard. RoomStats(ctx context.Context) (RoomStats, error) + + // WriteOutliers stores events as outliers in one transaction, skipping + // any already held. It is how a join over federation stores the room + // state and auth chain it is handed, which can run to tens of thousands + // of events: one Write per event would be one transaction per event. + WriteOutliers(ctx context.Context, roomID string, events []StoredEvent) error + + // WriteBackfill stores a room's history, fetched from another server, + // below everything the room already holds, in one transaction. + // + // The events are given oldest first and keep that order: each is + // numbered by a counter that runs downwards from -1, so the whole batch + // sorts below every event this server held when it arrived, and a + // later batch sorts below this one. An event already in the timeline is + // skipped; an outlier is promoted into the timeline in place, keeping + // its stored JSON. Nothing else about the room changes — not its + // current state, not its forward extremities, not its member counts — + // because history is by definition already reflected in them. + WriteBackfill(ctx context.Context, roomID string, events []BackfillEvent) error + + // InviteState returns the stripped state stored with a user's invite, + // or nil when there is none. + InviteState(ctx context.Context, roomID, userID string) ([]byte, error) + + // HasJoinedMemberOn reports whether any user of a server is joined to + // a room. It is asked about every event that arrives over federation — + // is this server still in the room, is the sender's? — and answering + // it by listing the members would read every row of a room of tens of + // thousands. + HasJoinedMemberOn(ctx context.Context, roomID, serverName string) (bool, error) + + // EventByID reads an event without knowing its room, returning + // ErrNoSuchEvent if it is absent. The federation API's /event names + // only the event; everything else asks within a room. + EventByID(ctx context.Context, eventID string) (StoredEvent, error) } diff --git a/server/internal/storage/rooms_test.go b/server/internal/storage/rooms_test.go index 9c7ea7f..9240989 100644 --- a/server/internal/storage/rooms_test.go +++ b/server/internal/storage/rooms_test.go @@ -41,7 +41,7 @@ func resetRoomTables(t *testing.T, driver string) { if _, err := db.ExecContext(t.Context(), `TRUNCATE roomserver_state_entries, roomserver_state_snapshots, roomserver_current_state, roomserver_extremities, - roomserver_transactions, roomserver_aliases, + roomserver_transactions, roomserver_aliases, roomserver_invite_state, roomserver_events, roomserver_rooms`); err != nil { t.Fatalf("reset the room tables: %v", err) } diff --git a/server/internal/storage/sqlite/migrations/0005_federation.sql b/server/internal/storage/sqlite/migrations/0005_federation.sql new file mode 100644 index 0000000..7d7732d --- /dev/null +++ b/server/internal/storage/sqlite/migrations/0005_federation.sql @@ -0,0 +1,113 @@ +-- Federation: other servers' keys, the outbound queue and its per-server +-- backoff, the inbound transaction log, and the two additions the room +-- tables need once a room's events can come from somewhere else. +-- +-- Timestamps are milliseconds since the Unix epoch, as everywhere else in +-- Warp. + +-- An outlier is an event this server holds without holding the room's +-- state at it: the state and auth chain a join is handed by the resident +-- server, an auth event fetched to judge something else, an invite to a +-- room nobody here is in, an event that soft-failed. It is a real event — +-- it is signed, it was authorised against its own auth events, and other +-- events can cite it — but it has no place in the timeline: no state +-- snapshot (state_before_id stays 0), never a forward extremity, never +-- returned by /sync or /messages. +-- +-- It goes on the events table rather than into a table of its own because +-- every read of a room's current state joins the events table, and the +-- current state of a room joined over federation is made almost entirely +-- of outliers. An outlier that later arrives as an ordinary event — fetched +-- by /get_missing_events, say — is promoted in place. +ALTER TABLE roomserver_events ADD COLUMN outlier BOOLEAN NOT NULL DEFAULT 0; + +-- The stripped state an invite over federation arrives with. +-- +-- An invite to a room this server is not in is the invite event alone: the +-- room's state is on the inviting server. What a client shows in its +-- invite list — the room's name, its avatar, who invited you — comes from +-- the invite_room_state the inviting server sent along, which is stored +-- here until the invite is answered and then deleted with it. +CREATE TABLE IF NOT EXISTS roomserver_invite_state ( + room_id TEXT NOT NULL REFERENCES roomserver_rooms (room_id) ON DELETE CASCADE, + user_id TEXT NOT NULL, + -- A JSON array of stripped state events, exactly as received. + state_json TEXT NOT NULL, + PRIMARY KEY (room_id, user_id) +); + +-- Other servers' signing keys, as their key servers or a notary reported +-- them. A key is never deleted: an event signed years ago is still +-- verified with the key that signed it, and a server that has rotated its +-- key lists the old one only as an old_verify_key with the time it expired. +CREATE TABLE IF NOT EXISTS federation_server_keys ( + server_name TEXT NOT NULL, + key_id TEXT NOT NULL, + -- Unpadded base64, as it is on the wire. + public_key TEXT NOT NULL, + -- How long the server vouched for the key when it was fetched. A key + -- is re-fetched once this passes, and an event from room version 5 on + -- may not be verified with a key whose validity ended before the event + -- was sent. + valid_until_ts BIGINT NOT NULL, + -- When the key stopped being used, for an old_verify_key; 0 for a key + -- the server still signs with. + expired_ts BIGINT NOT NULL DEFAULT 0, + fetched_at_ms BIGINT NOT NULL, + PRIMARY KEY (server_name, key_id) +); + +-- One counter row per stream; see roomserver_sequences for why these are +-- rows and not database sequences. +CREATE TABLE IF NOT EXISTS federation_sequences ( + name TEXT NOT NULL PRIMARY KEY, + value BIGINT NOT NULL +); + +INSERT INTO federation_sequences (name, value) VALUES ('queue', 0) + ON CONFLICT (name) DO NOTHING; + +-- The outbound queue: every PDU and EDU waiting for a destination. +-- +-- A row is deleted once the destination has acknowledged the transaction +-- that carried it, so what is here after a restart is exactly what still +-- has to be sent. The payload is the JSON that goes on the wire, copied at +-- enqueue time: re-reading a PDU from the room tables at send time would +-- send its redacted form if it was redacted in the meantime, and that is +-- not the event the destination has to verify. +CREATE TABLE IF NOT EXISTS federation_queue ( + queue_id BIGINT NOT NULL PRIMARY KEY, + destination TEXT NOT NULL, + -- 'pdu' or 'edu'. + kind TEXT NOT NULL, + room_id TEXT NOT NULL DEFAULT '', + event_id TEXT NOT NULL DEFAULT '', + payload TEXT NOT NULL, + queued_at_ms BIGINT NOT NULL +); + +CREATE INDEX IF NOT EXISTS federation_queue_destination_idx + ON federation_queue (destination, kind, queue_id); + +-- Per-destination delivery health: how often sending has failed in a row +-- and when it may next be tried. A destination with no row has never +-- failed. +CREATE TABLE IF NOT EXISTS federation_destinations ( + server_name TEXT NOT NULL PRIMARY KEY, + failures INTEGER NOT NULL DEFAULT 0, + retry_at_ms BIGINT NOT NULL DEFAULT 0, + last_success_ms BIGINT NOT NULL DEFAULT 0, + last_failure_ms BIGINT NOT NULL DEFAULT 0, + last_error TEXT NOT NULL DEFAULT '' +); + +-- Transactions received from other servers, so that a retransmission is +-- answered with the response the first delivery got instead of being +-- processed a second time. +CREATE TABLE IF NOT EXISTS federation_inbound_txns ( + origin TEXT NOT NULL, + txn_id TEXT NOT NULL, + response_json TEXT NOT NULL, + received_at_ms BIGINT NOT NULL, + PRIMARY KEY (origin, txn_id) +); diff --git a/server/internal/storage/sqlite/migrations/0006_backfill.sql b/server/internal/storage/sqlite/migrations/0006_backfill.sql new file mode 100644 index 0000000..b33e307 --- /dev/null +++ b/server/internal/storage/sqlite/migrations/0006_backfill.sql @@ -0,0 +1,15 @@ +-- Outbound backfill: history fetched from other servers for a room this +-- server joined part-way through. +-- +-- roomserver_events.stream_ordering is the order clients are shown a room +-- in, and the stream_ordering counter only ever goes up: every event it +-- numbers arrived after the ones before it. Backfilled history arrives +-- after everything too, but belongs before all of it. So it is numbered by +-- a second counter that runs the other way — from -1 downwards — and every +-- backfilled event sits below every event this server held when it came. +-- +-- Zero is neither counter's: the stream_ordering counter's first value is +-- 1 and this one's is -1, and a window bound of zero keeps meaning +-- "unbounded at that end". +INSERT INTO roomserver_sequences (name, value) VALUES ('backfill_ordering', 0) + ON CONFLICT (name) DO NOTHING; diff --git a/server/internal/storage/sqlstore/federation.go b/server/internal/storage/sqlstore/federation.go new file mode 100644 index 0000000..56f6d1a --- /dev/null +++ b/server/internal/storage/sqlstore/federation.go @@ -0,0 +1,323 @@ +package sqlstore + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Federation is the shared SQL implementation of storage.FederationStore. +type Federation struct { + db *sql.DB + rebind func(string) string +} + +// NewFederation builds a federation store. +func NewFederation(db *sql.DB, rebind func(string) string) *Federation { + if rebind == nil { + rebind = func(query string) string { return query } + } + return &Federation{db: db, rebind: rebind} +} + +// The sequence table of this component, and its one counter. +const ( + federationSequences = "federation_sequences" + seqQueue = "queue" +) + +func (s *Federation) q(query string) string { return s.rebind(query) } + +func (s *Federation) tx(ctx context.Context, fn func(*sql.Tx) error) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin transaction: %w", err) + } + defer func() { _ = tx.Rollback() }() + if err := fn(tx); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit transaction: %w", err) + } + return nil +} + +const upsertServerKeySQL = ` + INSERT INTO federation_server_keys + (server_name, key_id, public_key, valid_until_ts, expired_ts, fetched_at_ms) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (server_name, key_id) DO UPDATE SET + public_key = excluded.public_key, + valid_until_ts = excluded.valid_until_ts, + expired_ts = excluded.expired_ts, + fetched_at_ms = excluded.fetched_at_ms` + +// StoreServerKeys implements storage.FederationStore. +func (s *Federation) StoreServerKeys(ctx context.Context, keys []storage.ServerKey) error { + if len(keys) == 0 { + return nil + } + return s.tx(ctx, func(tx *sql.Tx) error { + for _, k := range keys { + if _, err := tx.ExecContext(ctx, s.q(upsertServerKeySQL), + k.ServerName, k.KeyID, k.PublicKey, k.ValidUntilTS, k.ExpiredTS, k.FetchedAtMS); err != nil { + return fmt.Errorf("store key %s of %s: %w", k.KeyID, k.ServerName, err) + } + } + return nil + }) +} + +// ServerKeys implements storage.FederationStore. +func (s *Federation) ServerKeys(ctx context.Context, serverName string, keyIDs []string) ([]storage.ServerKey, error) { + query := ` + SELECT server_name, key_id, public_key, valid_until_ts, expired_ts, fetched_at_ms + FROM federation_server_keys WHERE server_name = ?` + args := []any{serverName} + if len(keyIDs) > 0 { + query += ` AND key_id IN (` + placeholders(len(keyIDs)) + `)` + for _, id := range keyIDs { + args = append(args, id) + } + } + query += ` ORDER BY key_id` + + rows, err := s.db.QueryContext(ctx, s.q(query), args...) + if err != nil { + return nil, fmt.Errorf("read the keys of %q: %w", serverName, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.ServerKey + for rows.Next() { + var k storage.ServerKey + if err := rows.Scan(&k.ServerName, &k.KeyID, &k.PublicKey, + &k.ValidUntilTS, &k.ExpiredTS, &k.FetchedAtMS); err != nil { + return nil, fmt.Errorf("read the keys of %q: %w", serverName, err) + } + out = append(out, k) + } + return out, rows.Err() +} + +const insertQueuedSQL = ` + INSERT INTO federation_queue + (queue_id, destination, kind, room_id, event_id, payload, queued_at_ms) + VALUES (?, ?, ?, ?, ?, ?, ?)` + +// Enqueue implements storage.FederationStore. +func (s *Federation) Enqueue(ctx context.Context, items []storage.QueuedItem) ([]storage.QueuedItem, error) { + if len(items) == 0 { + return nil, nil + } + out := make([]storage.QueuedItem, len(items)) + copy(out, items) + err := s.tx(ctx, func(tx *sql.Tx) error { + top, err := advanceSequence(ctx, tx, s.q, federationSequences, seqQueue, int64(len(out))) + if err != nil { + return err + } + first := top - int64(len(out)) + 1 + for i := range out { + out[i].ID = first + int64(i) + item := out[i] + if _, err := tx.ExecContext(ctx, s.q(insertQueuedSQL), + item.ID, item.Destination, item.Kind, item.RoomID, item.EventID, + string(item.Payload), item.QueuedAtMS); err != nil { + return fmt.Errorf("queue for %s: %w", item.Destination, err) + } + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("enqueue %d items: %w", len(items), err) + } + return out, nil +} + +const selectQueuedSQL = ` + SELECT queue_id, destination, kind, room_id, event_id, payload, queued_at_ms + FROM federation_queue + WHERE destination = ? AND kind = ? + ORDER BY queue_id + LIMIT ?` + +// Queued implements storage.FederationStore. +func (s *Federation) Queued(ctx context.Context, destination, kind string, limit int) ([]storage.QueuedItem, error) { + if limit <= 0 { + return nil, nil + } + rows, err := s.db.QueryContext(ctx, s.q(selectQueuedSQL), destination, kind, limit) + if err != nil { + return nil, fmt.Errorf("read the queue for %q: %w", destination, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.QueuedItem + for rows.Next() { + var ( + item storage.QueuedItem + payload string + ) + if err := rows.Scan(&item.ID, &item.Destination, &item.Kind, &item.RoomID, + &item.EventID, &payload, &item.QueuedAtMS); err != nil { + return nil, fmt.Errorf("read the queue for %q: %w", destination, err) + } + item.Payload = []byte(payload) + out = append(out, item) + } + return out, rows.Err() +} + +// Dequeue implements storage.FederationStore. +func (s *Federation) Dequeue(ctx context.Context, ids []int64) error { + if len(ids) == 0 { + return nil + } + args := make([]any, len(ids)) + for i, id := range ids { + args[i] = id + } + query := `DELETE FROM federation_queue WHERE queue_id IN (` + placeholders(len(ids)) + `)` + if _, err := s.db.ExecContext(ctx, s.q(query), args...); err != nil { + return fmt.Errorf("dequeue %d items: %w", len(ids), err) + } + return nil +} + +const selectQueuedDestinationsSQL = ` + SELECT DISTINCT destination FROM federation_queue ORDER BY destination` + +// QueuedDestinations implements storage.FederationStore. +func (s *Federation) QueuedDestinations(ctx context.Context) ([]string, error) { + rows, err := s.db.QueryContext(ctx, s.q(selectQueuedDestinationsSQL)) + if err != nil { + return nil, fmt.Errorf("read the queued destinations: %w", err) + } + defer func() { _ = rows.Close() }() + + var out []string + for rows.Next() { + var d string + if err := rows.Scan(&d); err != nil { + return nil, fmt.Errorf("read the queued destinations: %w", err) + } + out = append(out, d) + } + return out, rows.Err() +} + +const destinationColumns = ` + server_name, failures, retry_at_ms, last_success_ms, last_failure_ms, last_error` + +func scanDestination(scan func(...any) error) (storage.Destination, error) { + var d storage.Destination + err := scan(&d.ServerName, &d.Failures, &d.RetryAtMS, &d.LastSuccessMS, &d.LastFailureMS, &d.LastError) + return d, err +} + +const selectDestinationSQL = `SELECT ` + destinationColumns + ` + FROM federation_destinations WHERE server_name = ?` + +// Destination implements storage.FederationStore. +func (s *Federation) Destination(ctx context.Context, serverName string) (storage.Destination, error) { + d, err := scanDestination(s.db.QueryRowContext(ctx, s.q(selectDestinationSQL), serverName).Scan) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.Destination{ServerName: serverName}, nil + case err != nil: + return storage.Destination{}, fmt.Errorf("read destination %q: %w", serverName, err) + } + return d, nil +} + +const upsertDestinationSQL = ` + INSERT INTO federation_destinations (` + destinationColumns + `) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (server_name) DO UPDATE SET + failures = excluded.failures, + retry_at_ms = excluded.retry_at_ms, + last_success_ms = excluded.last_success_ms, + last_failure_ms = excluded.last_failure_ms, + last_error = excluded.last_error` + +// SetDestination implements storage.FederationStore. +func (s *Federation) SetDestination(ctx context.Context, d storage.Destination) error { + if _, err := s.db.ExecContext(ctx, s.q(upsertDestinationSQL), + d.ServerName, d.Failures, d.RetryAtMS, d.LastSuccessMS, d.LastFailureMS, d.LastError); err != nil { + return fmt.Errorf("record destination %q: %w", d.ServerName, err) + } + return nil +} + +// selectDestinationSummariesSQL lists every server that either has a +// health row or something queued. The union is what makes a destination +// that has never failed, but has a backlog, show up at all. +const selectDestinationSummariesSQL = ` + SELECT n.server_name, + COALESCE(d.failures, 0), COALESCE(d.retry_at_ms, 0), + COALESCE(d.last_success_ms, 0), COALESCE(d.last_failure_ms, 0), + COALESCE(d.last_error, ''), + (SELECT COUNT(*) FROM federation_queue q WHERE q.destination = n.server_name AND q.kind = 'pdu'), + (SELECT COUNT(*) FROM federation_queue q WHERE q.destination = n.server_name AND q.kind = 'edu') + FROM ( + SELECT server_name FROM federation_destinations + UNION + SELECT DISTINCT destination FROM federation_queue + ) n + LEFT JOIN federation_destinations d ON d.server_name = n.server_name + ORDER BY n.server_name` + +// Destinations implements storage.FederationStore. +func (s *Federation) Destinations(ctx context.Context) ([]storage.DestinationSummary, error) { + rows, err := s.db.QueryContext(ctx, s.q(selectDestinationSummariesSQL)) + if err != nil { + return nil, fmt.Errorf("read the destinations: %w", err) + } + defer func() { _ = rows.Close() }() + + var out []storage.DestinationSummary + for rows.Next() { + var d storage.DestinationSummary + if err := rows.Scan(&d.ServerName, &d.Failures, &d.RetryAtMS, &d.LastSuccessMS, + &d.LastFailureMS, &d.LastError, &d.QueuedPDUs, &d.QueuedEDUs); err != nil { + return nil, fmt.Errorf("read the destinations: %w", err) + } + out = append(out, d) + } + return out, rows.Err() +} + +const selectInboundTxnSQL = ` + SELECT response_json FROM federation_inbound_txns WHERE origin = ? AND txn_id = ?` + +// InboundTransaction implements storage.FederationStore. +func (s *Federation) InboundTransaction(ctx context.Context, origin, txnID string) ([]byte, bool, error) { + var response string + err := s.db.QueryRowContext(ctx, s.q(selectInboundTxnSQL), origin, txnID).Scan(&response) + switch { + case errors.Is(err, sql.ErrNoRows): + return nil, false, nil + case err != nil: + return nil, false, fmt.Errorf("read transaction %q from %q: %w", txnID, origin, err) + } + return []byte(response), true, nil +} + +const insertInboundTxnSQL = ` + INSERT INTO federation_inbound_txns (origin, txn_id, response_json, received_at_ms) + VALUES (?, ?, ?, ?) + ON CONFLICT (origin, txn_id) DO NOTHING` + +// RecordInboundTransaction implements storage.FederationStore. +func (s *Federation) RecordInboundTransaction(ctx context.Context, origin, txnID string, response []byte, atMS int64) error { + if _, err := s.db.ExecContext(ctx, s.q(insertInboundTxnSQL), + origin, txnID, string(response), atMS); err != nil { + return fmt.Errorf("record transaction %q from %q: %w", txnID, origin, err) + } + return nil +} diff --git a/server/internal/storage/sqlstore/rooms.go b/server/internal/storage/sqlstore/rooms.go index 21caa91..1000fba 100644 --- a/server/internal/storage/sqlstore/rooms.go +++ b/server/internal/storage/sqlstore/rooms.go @@ -46,10 +46,14 @@ const ( maxRoomPageSize = 1000 ) -// The two counters in roomserver_sequences. +// The counters in roomserver_sequences. const ( seqStreamOrdering = "stream_ordering" seqStateSnapshot = "state_snapshot" + + // seqBackfillOrdering numbers backfilled history. It starts at 0 and + // only ever goes down; see WriteBackfill. + seqBackfillOrdering = "backfill_ordering" ) // q rebinds a statement for this driver. @@ -121,14 +125,22 @@ func (s *Rooms) Write(ctx context.Context, w storage.RoomWrite) (int64, error) { return err } - snapshotID, err := s.stateSnapshot(ctx, tx, w.Event.RoomID, w.StateBefore) - if err != nil { - return err + // An outlier has no state snapshot: that is what makes it one. + var snapshotID int64 + if !w.Outlier { + if snapshotID, err = s.stateSnapshot(ctx, tx, w.Event.RoomID, w.StateBefore); err != nil { + return err + } } - if err := s.insertEvent(ctx, tx, w.Event, ordering, snapshotID); err != nil { + if err := s.insertEvent(ctx, tx, w.Event, ordering, snapshotID, w.Outlier); err != nil { return err } + if w.Event.Type == memberEventType && w.Event.StateKey != nil { + if err := s.replaceInviteState(ctx, tx, w.Event.RoomID, *w.Event.StateKey, w.InviteState); err != nil { + return err + } + } if w.Redacts != "" { if err := s.applyRedaction(ctx, tx, w.Event.RoomID, w.Redacts, w.Event.EventID, w.RedactedJSON); err != nil { @@ -246,21 +258,55 @@ func stateHash(entries []storage.StateEntry) string { return base64.RawURLEncoding.EncodeToString(h.Sum(nil)) } -const insertEventSQL = ` - INSERT INTO roomserver_events ( - event_id, room_id, stream_ordering, event_type, state_key, - sender, depth, origin_server_ts, event_json, state_before_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` +// memberEventType is the one event type the store has to recognise: its +// rows carry a membership, and its arrival clears a stored invite. +const memberEventType = "m.room.member" -func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEvent, ordering, snapshotID int64) error { - _, err := tx.ExecContext(ctx, s.q(insertEventSQL), +const ( + insertEventSQL = ` + INSERT INTO roomserver_events ( + event_id, room_id, stream_ordering, event_type, state_key, + sender, depth, origin_server_ts, event_json, state_before_id, outlier) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + selectOutlierSQL = `SELECT outlier FROM roomserver_events WHERE event_id = ?` + promoteOutlierSQL = ` + UPDATE roomserver_events + SET stream_ordering = ?, state_before_id = ?, outlier = ? + WHERE event_id = ? AND outlier = ?` +) + +// insertEvent writes the event row. +// +// An event this server already holds is not written twice. The one +// exception is an outlier arriving again as an ordinary event, which is +// promoted in place: it gets the new stream position and the state +// snapshot it now has, and joins the timeline. Its JSON is left as it was — +// the event ID says the two copies are the same event, and the stored one +// may already have been redacted. +func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEvent, ordering, snapshotID int64, outlier bool) error { + var existingOutlier bool + err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), e.EventID).Scan(&existingOutlier) + switch { + case err == nil: + if existingOutlier && !outlier { + if _, err := tx.ExecContext(ctx, s.q(promoteOutlierSQL), + ordering, snapshotID, false, e.EventID, true); err != nil { + return fmt.Errorf("promote outlier: %w", err) + } + } + // The same event twice is not a failure worth propagating: an + // event ID is a hash of its contents, so the second copy says + // exactly what the first one did. + return nil + case !errors.Is(err, sql.ErrNoRows): + return fmt.Errorf("look up event: %w", err) + } + + _, err = tx.ExecContext(ctx, s.q(insertEventSQL), e.EventID, e.RoomID, ordering, e.Type, e.StateKey, - e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID) + e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, outlier) if err != nil { if s.isUnique(err) { - // The same event twice is not a failure worth propagating: an - // event ID is a hash of its contents, so the second copy says - // exactly what the first one did. return nil } return fmt.Errorf("insert event: %w", err) @@ -268,6 +314,147 @@ func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEve return nil } +const ( + deleteInviteStateSQL = `DELETE FROM roomserver_invite_state WHERE room_id = ? AND user_id = ?` + insertInviteStateSQL = ` + INSERT INTO roomserver_invite_state (room_id, user_id, state_json) VALUES (?, ?, ?)` +) + +// replaceInviteState clears a user's stored invite state and, for an +// invite that came with some, stores the new one. +func (s *Rooms) replaceInviteState(ctx context.Context, tx *sql.Tx, roomID, userID string, state []byte) error { + if _, err := tx.ExecContext(ctx, s.q(deleteInviteStateSQL), roomID, userID); err != nil { + return fmt.Errorf("clear invite state: %w", err) + } + if len(state) == 0 { + return nil + } + if _, err := tx.ExecContext(ctx, s.q(insertInviteStateSQL), roomID, userID, string(state)); err != nil { + return fmt.Errorf("store invite state: %w", err) + } + return nil +} + +const selectInviteStateSQL = ` + SELECT state_json FROM roomserver_invite_state WHERE room_id = ? AND user_id = ?` + +// InviteState implements storage.RoomStore. +func (s *Rooms) InviteState(ctx context.Context, roomID, userID string) ([]byte, error) { + var state string + err := s.db.QueryRowContext(ctx, s.q(selectInviteStateSQL), roomID, userID).Scan(&state) + switch { + case errors.Is(err, sql.ErrNoRows): + return nil, nil + case err != nil: + return nil, fmt.Errorf("read the invite state of %q in %q: %w", userID, roomID, err) + } + return []byte(state), nil +} + +// WriteOutliers implements storage.RoomStore. +// +// The stream positions are taken as one block, in one UPDATE, rather than +// one at a time: a join to a large room hands over tens of thousands of +// events, and a counter bumped per event would be that many statements +// holding the counter's row lock. +func (s *Rooms) WriteOutliers(ctx context.Context, roomID string, events []storage.StoredEvent) error { + if len(events) == 0 { + return nil + } + err := s.tx(ctx, func(tx *sql.Tx) error { + top, err := advanceSequence(ctx, tx, s.q, roomSequences, seqStreamOrdering, int64(len(events))) + if err != nil { + return err + } + first := top - int64(len(events)) + 1 + for i, e := range events { + if e.RoomID == "" { + e.RoomID = roomID + } + if err := s.insertEvent(ctx, tx, e, first+int64(i), 0, true); err != nil { + return fmt.Errorf("outlier %s: %w", e.EventID, err) + } + } + return nil + }) + if err != nil { + return fmt.Errorf("write %d outliers to %q: %w", len(events), roomID, err) + } + return nil +} + +// WriteBackfill implements storage.RoomStore. +// +// The positions come from their own counter, advanced downwards by the +// number of events actually written, in one UPDATE: the events already in +// the timeline are sorted out first, inside the transaction, so that the +// block taken is exactly as big as what goes into it. A hole below the +// room would be harmless — nothing streams through backfilled history, it +// is only ever read by bounds — but there is no reason to leave one. +// +// The batch takes the block oldest first: if the counter stood at -10 and +// three events are written, they become -13, -12 and -11. +func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []storage.BackfillEvent) error { + if len(events) == 0 { + return nil + } + err := s.tx(ctx, func(tx *sql.Tx) error { + // Which of the batch is new, which is an outlier to promote, and + // which is already in the timeline and left alone. + var write []storage.BackfillEvent + promote := map[string]bool{} + for _, b := range events { + var outlier bool + err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), b.Event.EventID).Scan(&outlier) + switch { + case errors.Is(err, sql.ErrNoRows): + write = append(write, b) + case err != nil: + return fmt.Errorf("look up event %s: %w", b.Event.EventID, err) + case outlier: + write = append(write, b) + promote[b.Event.EventID] = true + } + } + if len(write) == 0 { + return nil + } + + bottom, err := advanceSequence(ctx, tx, s.q, roomSequences, seqBackfillOrdering, -int64(len(write))) + if err != nil { + return err + } + for i, b := range write { + e := b.Event + if e.RoomID == "" { + e.RoomID = roomID + } + ordering := bottom + int64(i) + snapshotID, err := s.stateSnapshot(ctx, tx, e.RoomID, b.StateBefore) + if err != nil { + return err + } + if promote[e.EventID] { + if _, err := tx.ExecContext(ctx, s.q(promoteOutlierSQL), + ordering, snapshotID, false, e.EventID, true); err != nil { + return fmt.Errorf("promote outlier %s: %w", e.EventID, err) + } + continue + } + if _, err := tx.ExecContext(ctx, s.q(insertEventSQL), + e.EventID, e.RoomID, ordering, e.Type, e.StateKey, + e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, false); err != nil { + return fmt.Errorf("insert event %s: %w", e.EventID, err) + } + } + return nil + }) + if err != nil { + return fmt.Errorf("backfill %d events into %q: %w", len(events), roomID, err) + } + return nil +} + const applyRedactionSQL = ` UPDATE roomserver_events SET event_json = ?, redacted_by = ? WHERE room_id = ? AND event_id = ? AND redacted_by = ''` @@ -387,7 +574,7 @@ func (s *Rooms) recordTransaction(ctx context.Context, tx *sql.Tx, ref storage.T // scanEvent expects. const eventColumns = ` e.room_id, e.event_id, e.event_type, e.state_key, e.sender, - e.depth, e.origin_server_ts, e.event_json, e.stream_ordering, e.redacted_by` + e.depth, e.origin_server_ts, e.event_json, e.stream_ordering, e.redacted_by, e.outlier` // scanEvent reads one row of eventColumns. func scanEvent(scan func(...any) error) (storage.StoredEvent, error) { @@ -397,7 +584,7 @@ func scanEvent(scan func(...any) error) (storage.StoredEvent, error) { body string ) err := scan(&e.RoomID, &e.EventID, &e.Type, &stateKey, &e.Sender, - &e.Depth, &e.OriginServerTS, &body, &e.StreamOrdering, &e.RedactedBy) + &e.Depth, &e.OriginServerTS, &body, &e.StreamOrdering, &e.RedactedBy, &e.Outlier) if err != nil { return storage.StoredEvent{}, err } @@ -892,6 +1079,7 @@ var purgeStatements = []string{ `DELETE FROM roomserver_extremities WHERE room_id = ?`, `DELETE FROM roomserver_transactions WHERE room_id = ?`, `DELETE FROM roomserver_aliases WHERE room_id = ?`, + `DELETE FROM roomserver_invite_state WHERE room_id = ?`, `DELETE FROM roomserver_events WHERE room_id = ?`, `DELETE FROM roomserver_rooms WHERE room_id = ?`, } @@ -968,3 +1156,42 @@ func expectOneRow(result sql.Result, subject string, notFound error) error { } return nil } + +var selectEventByIDSQL = ` + SELECT ` + eventColumns + ` + FROM roomserver_events e WHERE e.event_id = ?` + +// EventByID implements storage.RoomStore. +func (s *Rooms) EventByID(ctx context.Context, eventID string) (storage.StoredEvent, error) { + e, err := scanEvent(s.db.QueryRowContext(ctx, s.q(selectEventByIDSQL), eventID).Scan) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.StoredEvent{}, fmt.Errorf("event %q: %w", eventID, storage.ErrNoSuchEvent) + case err != nil: + return storage.StoredEvent{}, fmt.Errorf("read event %q: %w", eventID, err) + } + return e, nil +} + +const selectJoinedMemberOnSQL = ` + SELECT 1 FROM roomserver_current_state + WHERE room_id = ? AND event_type = 'm.room.member' AND membership = 'join' + AND state_key LIKE ? ESCAPE '\' + LIMIT 1` + +// HasJoinedMemberOn implements storage.RoomStore. +// +// A user ID's server is everything after its first colon, and a localpart +// cannot contain one, so "ends with ':' plus the server name" is exactly +// "belongs to that server". +func (s *Rooms) HasJoinedMemberOn(ctx context.Context, roomID, serverName string) (bool, error) { + var one int + err := s.db.QueryRowContext(ctx, s.q(selectJoinedMemberOnSQL), roomID, `%:`+escapeLike(serverName)).Scan(&one) + switch { + case errors.Is(err, sql.ErrNoRows): + return false, nil + case err != nil: + return false, fmt.Errorf("look for members of %q in %q: %w", serverName, roomID, err) + } + return true, nil +} diff --git a/server/internal/storage/sqlstore/sequences.go b/server/internal/storage/sqlstore/sequences.go index 5f0e560..950b26f 100644 --- a/server/internal/storage/sqlstore/sequences.go +++ b/server/internal/storage/sqlstore/sequences.go @@ -44,8 +44,15 @@ type execQuerier interface { // statements are two transactions, and two callers can then be handed the // same value. func nextSequence(ctx context.Context, db execQuerier, q func(string) string, table, name string) (int64, error) { - bump := fmt.Sprintf(`UPDATE %s SET value = value + 1 WHERE name = ?`, table) - if _, err := db.ExecContext(ctx, q(bump), name); err != nil { + return advanceSequence(ctx, db, q, table, name, 1) +} + +// advanceSequence hands out a block of n values and returns the last of +// them; the block is the n values ending there. Like nextSequence it must +// be called inside a transaction. +func advanceSequence(ctx context.Context, db execQuerier, q func(string) string, table, name string, n int64) (int64, error) { + bump := fmt.Sprintf(`UPDATE %s SET value = value + ? WHERE name = ?`, table) + if _, err := db.ExecContext(ctx, q(bump), n, name); err != nil { return 0, fmt.Errorf("advance the %s counter: %w", name, err) } return currentSequence(ctx, db, q, table, name) diff --git a/server/internal/storage/sqlstore/sqlstore.go b/server/internal/storage/sqlstore/sqlstore.go index 3c41ff8..32003ec 100644 --- a/server/internal/storage/sqlstore/sqlstore.go +++ b/server/internal/storage/sqlstore/sqlstore.go @@ -59,6 +59,7 @@ type Store struct { users *Users rooms *Rooms sync *Sync + fed *Federation } // New builds a Store. It does not touch the database. @@ -77,6 +78,7 @@ func New(opts Options) (*Store, error) { users: NewUsers(opts.DB, opts.Rebind, opts.IsUniqueViolation), rooms: NewRooms(opts.DB, opts.Rebind, opts.IsUniqueViolation), sync: NewSync(opts.DB, opts.Rebind, opts.IsUniqueViolation), + fed: NewFederation(opts.DB, opts.Rebind), }, nil } @@ -104,6 +106,9 @@ func (s *Store) Rooms() storage.RoomStore { return s.rooms } // Sync implements storage.Database. func (s *Store) Sync() storage.SyncStore { return s.sync } +// Federation implements storage.Database. +func (s *Store) Federation() storage.FederationStore { return s.fed } + // Migrate implements storage.Database. func (s *Store) Migrate(ctx context.Context) error { ms, err := s.migrations() diff --git a/server/internal/storage/sqlstore/timeline.go b/server/internal/storage/sqlstore/timeline.go index 0bad34f..dbe689d 100644 --- a/server/internal/storage/sqlstore/timeline.go +++ b/server/internal/storage/sqlstore/timeline.go @@ -75,7 +75,7 @@ func (s *Rooms) MembershipsForUser(ctx context.Context, userID string, membershi var selectUserMembershipEventsSQL = ` SELECT ` + eventColumns + ` FROM roomserver_events e - WHERE e.room_id = ? AND e.event_type = 'm.room.member' AND e.state_key = ? + WHERE e.room_id = ? AND e.event_type = 'm.room.member' AND e.state_key = ? AND NOT e.outlier ORDER BY e.stream_ordering` // UserMembershipEvents implements storage.RoomStore. @@ -97,13 +97,17 @@ func (s *Rooms) Timeline(ctx context.Context, req storage.TimelineRequest) ([]st return nil, nil } - conditions := []string{`e.room_id = ?`} + // Outliers have no place in a timeline; see the outlier column. + conditions := []string{`e.room_id = ?`, `NOT e.outlier`} args := []any{req.RoomID} - if req.Low > 0 { + // Zero is "unbounded", not a position, at either end: a bound can be + // negative, inside backfilled history, so "is it set" cannot be + // spelled "is it positive". + if req.Low != 0 { conditions = append(conditions, `e.stream_ordering > ?`) args = append(args, req.Low) } - if req.High > 0 { + if req.High != 0 { conditions = append(conditions, `e.stream_ordering <= ?`) args = append(args, req.High) } @@ -229,13 +233,16 @@ func (s *Rooms) RoomsWithEvents(ctx context.Context, roomIDs []string, from, to query := ` SELECT DISTINCT room_id FROM roomserver_events - WHERE room_id IN (` + placeholders(len(roomIDs)) + `) AND stream_ordering > ?` + WHERE room_id IN (` + placeholders(len(roomIDs)) + `) AND NOT outlier` args := make([]any, 0, len(roomIDs)+2) for _, id := range roomIDs { args = append(args, id) } - args = append(args, from) - if to > 0 { + if from != 0 { + query += ` AND stream_ordering > ?` + args = append(args, from) + } + if to != 0 { query += ` AND stream_ordering <= ?` args = append(args, to) } @@ -261,7 +268,7 @@ func (s *Rooms) RoomsWithEvents(ctx context.Context, roomIDs []string, from, to const selectEventAtStreamSQL = ` SELECT e.event_id, e.event_type, e.state_key, e.state_before_id FROM roomserver_events e - WHERE e.room_id = ? AND e.stream_ordering <= ? + WHERE e.room_id = ? AND NOT e.outlier AND e.stream_ordering <= ? ORDER BY e.stream_ordering DESC LIMIT 1` @@ -278,7 +285,7 @@ const selectSnapshotEntriesSQL = ` // the snapshot is the state the event was authorised against, which is the // state *before* it. func (s *Rooms) StateAtStream(ctx context.Context, roomID string, pos int64) ([]storage.StateEntry, error) { - if pos <= 0 { + if pos == 0 { return nil, nil } diff --git a/server/internal/storage/storage.go b/server/internal/storage/storage.go index 7638f9f..d0f1743 100644 --- a/server/internal/storage/storage.go +++ b/server/internal/storage/storage.go @@ -71,6 +71,10 @@ type Database interface { // uploaded filters. Sync() SyncStore + // Federation is the federation store: other servers' keys, the + // outbound queue and its backoff, and the inbound transaction log. + Federation() FederationStore + // Close releases the connection pool. Close() error } diff --git a/server/internal/storage/sync.go b/server/internal/storage/sync.go index f0d9dda..e735cbc 100644 --- a/server/internal/storage/sync.go +++ b/server/internal/storage/sync.go @@ -59,10 +59,17 @@ const ( // lets a token a client was handed be passed straight back without // re-reading the event it names: a position is somewhere a client has got // to, not an event it still wants. +// +// Positions can be negative: history backfilled from other servers is +// numbered downwards from -1, below everything that was already here (see +// RoomStore.WriteBackfill). Zero is never a position, which is what lets +// it go on meaning "unbounded" here — and why the position just below an +// event is PositionBefore and not a plain subtraction. type TimelineRequest struct { RoomID string - // Low is an exclusive lower bound. Zero means the start of the room. + // Low is an exclusive lower bound. Zero means unbounded: the start of + // the room, backfilled history included. Low int64 // High is an inclusive upper bound. Zero means the top of the stream, @@ -84,6 +91,21 @@ type TimelineRequest struct { Filter TimelineFilter } +// PositionBefore is the position just below pos: the bound that names +// "everything up to, but not including, the event at pos". +// +// It is pos-1 except at 1, whose predecessor is -1. Zero is not a +// position — it is the "unbounded" bound of every window — so a token +// below the first event this server ever persisted has to skip it, or +// handing that token back as a High bound would read the whole room again +// instead of the backfilled history beneath it. +func PositionBefore(pos int64) int64 { + if pos == 1 { + return -1 + } + return pos - 1 +} + // TimelineFilter is the part of a Client-Server API event filter that the // store can apply itself. // diff --git a/server/internal/syncapi/context.go b/server/internal/syncapi/context.go index b424676..c8d606d 100644 --- a/server/internal/syncapi/context.go +++ b/server/internal/syncapi/context.go @@ -106,7 +106,7 @@ func (s *API) Context(ctx context.Context, req ContextRequest) (ContextResponse, User: req.User, // Strictly before the target: High is inclusive, and the // target itself is reported separately. - High: position - 1, + High: storage.PositionBefore(position), Direction: storage.DirectionBackward, Limit: before, Filter: filter.TimelineFilter(), @@ -120,7 +120,7 @@ func (s *API) Context(ctx context.Context, req ContextRequest) (ContextResponse, out.Start = pduToken(earlier.Start).String() } if out.Start == "" { - out.Start = pduToken(position - 1).String() + out.Start = pduToken(storage.PositionBefore(position)).String() } if after > 0 { diff --git a/server/internal/syncapi/messages.go b/server/internal/syncapi/messages.go index 9ee2c07..acea0f8 100644 --- a/server/internal/syncapi/messages.go +++ b/server/internal/syncapi/messages.go @@ -112,6 +112,9 @@ func (s *API) Messages(ctx context.Context, req MessagesRequest) (MessagesRespon Direction: direction, Limit: limit, Filter: filter.TimelineFilter(), + // Scrolling up past the oldest event this server holds of a room + // joined over federation asks the room's other servers for more. + Backfill: true, }) if err != nil { return MessagesResponse{}, err diff --git a/server/internal/syncapi/outbound.go b/server/internal/syncapi/outbound.go new file mode 100644 index 0000000..35a66be --- /dev/null +++ b/server/internal/syncapi/outbound.go @@ -0,0 +1,31 @@ +package syncapi + +import ( + "context" + "encoding/json" + "time" +) + +// Outbound passes this server's ephemeral events on to other servers. +// +// Typing notifications, read receipts and to-device messages are not room +// events and never go through the roomserver, so the federation queue +// cannot pick them up from there; the sync API, which is where they are +// written, hands them over instead. Only what a local user did is handed +// over — the same calls are how an EDU from another server is taken in, +// and passing those on again would echo them back to where they came +// from. +// +// Every method must return quickly: they are called on the request path +// of the client that typed, read or sent. +type Outbound interface { + Typing(ctx context.Context, roomID, userID string, typing bool, timeout time.Duration) + Receipt(ctx context.Context, roomID, userID, receiptType, eventID, threadID string, ts int64) + ToDevice(ctx context.Context, sender, eventType string, messages map[string]map[string]json.RawMessage) +} + +// SetOutbound connects the federation queue. It is a setter rather than an +// option because the queue is built after the sync API's dependencies and +// before the sync API is serving, and nil — no federation — is a valid +// value throughout. +func (s *API) SetOutbound(o Outbound) { s.outbound = o } diff --git a/server/internal/syncapi/presence.go b/server/internal/syncapi/presence.go index d316c3d..54518a5 100644 --- a/server/internal/syncapi/presence.go +++ b/server/internal/syncapi/presence.go @@ -104,7 +104,7 @@ func (s *API) Presence(ctx context.Context, userID string) (storage.PresenceStat return storage.PresenceState{}, ErrPresenceDisabled } if !s.isLocal(userID) { - // Remote presence needs federation (Phase 5). Reporting offline + // Remote presence is not fetched over federation. Reporting offline // would be a guess; saying so is not. return storage.PresenceState{}, errForbiddenf( "%s belongs to another server and federation is required to ask it", userID) diff --git a/server/internal/syncapi/receipts.go b/server/internal/syncapi/receipts.go index aaef608..4a01c8b 100644 --- a/server/internal/syncapi/receipts.go +++ b/server/internal/syncapi/receipts.go @@ -104,6 +104,10 @@ func (s *API) SetReceipt(ctx context.Context, req ReceiptRequest) error { return nil } s.notifier.NotifyRoom(req.RoomID) + // A private receipt never leaves the server, which is the point of it. + if s.outbound != nil && s.isLocal(req.User) { + s.outbound.Receipt(ctx, req.RoomID, req.User, req.ReceiptType, req.EventID, req.ThreadID, s.nowMS()) + } return nil } diff --git a/server/internal/syncapi/syncapi.go b/server/internal/syncapi/syncapi.go index 08ec4e6..0475a49 100644 --- a/server/internal/syncapi/syncapi.go +++ b/server/internal/syncapi/syncapi.go @@ -90,6 +90,10 @@ type API struct { // asking, where a clear refusal tells it this one has it turned off. presenceEnabled bool + // outbound passes local typing, receipts and remote to-device + // messages to the federation queue; nil without federation. + outbound Outbound + // longPollTimeout caps how long a /sync may block, whatever ?timeout= // the client asked for. A client asking for ten minutes would hold a // connection and a database handle for ten minutes. diff --git a/server/internal/syncapi/todevice.go b/server/internal/syncapi/todevice.go index d84e1b2..62f707d 100644 --- a/server/internal/syncapi/todevice.go +++ b/server/internal/syncapi/todevice.go @@ -33,13 +33,13 @@ type SendToDeviceRequest struct { // SendToDevice queues messages for delivery. // -// A message to a user of another server is dropped rather than refused. -// That is deliberate: a client encrypting for a room with remote members -// sends one message per device in one request, and failing the request -// because federation is off would stop it from delivering to the local -// devices as well. The dropped recipients simply never see the key and -// their client reports an undecryptable message, which is what they would -// see anyway. Phase 5 routes them. +// A message to a user of another server goes to the federation queue. With +// federation off it is dropped rather than refused. That is deliberate: a +// client encrypting for a room with remote members sends one message per +// device in one request, and failing the request because federation is +// off would stop it from delivering to the local devices as well. The +// dropped recipients simply never see the key and their client reports an +// undecryptable message, which is what they would see anyway. func (s *API) SendToDevice(ctx context.Context, req SendToDeviceRequest) error { if req.EventType == "" { return errInvalidParamf("an event type is required") @@ -48,11 +48,19 @@ func (s *API) SendToDevice(ctx context.Context, req SendToDeviceRequest) error { var ( queued []storage.NewToDeviceMessage woken []string + remote = map[string]map[string]json.RawMessage{} sentAt = s.nowMS() ) for userID, byDevice := range req.Messages { localpart, ok := s.localpartOf(userID) if !ok { + // A remote recipient goes to the federation queue when there + // is one and the sender is ours; a remote sender's message to + // a remote user is not this server's to forward. + if s.outbound != nil && s.isLocal(req.Sender) { + remote[userID] = byDevice + continue + } s.log.Debug("dropping a to-device message for a remote user", "sender", req.Sender, "recipient", userID) continue @@ -95,6 +103,9 @@ func (s *API) SendToDevice(ctx context.Context, req SendToDeviceRequest) error { woken = append(woken, userID) } + if len(remote) > 0 { + s.outbound.ToDevice(ctx, req.Sender, req.EventType, remote) + } if len(queued) == 0 { return nil } diff --git a/server/internal/syncapi/token.go b/server/internal/syncapi/token.go index 4d5a1b1..d493449 100644 --- a/server/internal/syncapi/token.go +++ b/server/internal/syncapi/token.go @@ -96,7 +96,10 @@ func ParseToken(raw string) (Token, error) { var positions [5]int64 for i, field := range fields { value, err := strconv.ParseInt(field, 10, 64) - if err != nil || value < 0 { + // The PDU position alone may be negative: /messages hands out + // positions inside history backfilled from other servers, which + // is numbered downwards from -1. Every other stream starts at 0. + if err != nil || (value < 0 && i > 0) { return Token{}, fmt.Errorf("%q is not a sync token: position %d is %q", raw, i+1, field) } positions[i] = value diff --git a/server/internal/syncapi/token_test.go b/server/internal/syncapi/token_test.go index 88dfd38..1dba487 100644 --- a/server/internal/syncapi/token_test.go +++ b/server/internal/syncapi/token_test.go @@ -20,6 +20,13 @@ func TestTokenRoundTrips(t *testing.T) { syncapi.Token{PDU: 1, Receipt: 2, ToDevice: 3, AccountData: 4, Presence: 5}, "s1_2_3_4_5", }, + { + // /messages pages back into history backfilled from other + // servers, which is numbered downwards from -1. + "a position inside backfilled history", + syncapi.Token{PDU: -7}, + "s-7", + }, { // A zero in the middle has to be written, because the // positions are read by place. @@ -77,7 +84,7 @@ func TestParseTokenRejectsNonsense(t *testing.T) { "nonsense", // no prefix "t1_2", // a prefix this build does not write "s", // a prefix and nothing else - "s-1", // a negative position + "s1_-1", // a negative position outside the event stream "sx", // not a number "s1_", // a trailing separator "s1_2_3_4_5_6", // more positions than there are streams diff --git a/server/internal/syncapi/typing.go b/server/internal/syncapi/typing.go index 2ff3b55..98564e1 100644 --- a/server/internal/syncapi/typing.go +++ b/server/internal/syncapi/typing.go @@ -196,6 +196,9 @@ func (s *API) SetTyping(ctx context.Context, req TypingRequest) error { } s.typing.set(req.RoomID, req.User, req.Typing, req.Timeout) + if s.outbound != nil && s.isLocal(req.User) { + s.outbound.Typing(ctx, req.RoomID, req.User, req.Typing, req.Timeout) + } return nil } diff --git a/server/test/complement/Dockerfile b/server/test/complement/Dockerfile index 2ac0208..9b0cfe7 100644 --- a/server/test/complement/Dockerfile +++ b/server/test/complement/Dockerfile @@ -16,10 +16,10 @@ # # docker build -t complement-warp -f server/test/complement/Dockerfile . # -# Federation (port 8448, served over TLS with a certificate signed by the CA -# Complement mounts at /complement/ca) arrives in Phase 5. Until then this -# image is for the federation=off profile: the C-S tests. The port is still -# exposed, because Complement expects it to be. +# Federation is served on 8448 over TLS, with a certificate the entrypoint +# has the CA Complement mounts at /complement/ca sign. The federation +# profile of run.sh switches it on; the default profile leaves it off and +# runs the C-S tests only. FROM golang:1.26-alpine AS build @@ -38,9 +38,9 @@ RUN go build -trimpath -o /out/warp ./cmd/warp FROM alpine:3.21 -# curl is what the health check uses; ca-certificates is for the outbound -# federation requests of Phase 5. -RUN apk add --no-cache ca-certificates curl +# curl is what the health check uses, ca-certificates the roots outbound +# federation trusts, and openssl signs the federation certificate. +RUN apk add --no-cache ca-certificates curl openssl COPY --from=build /out/warp /usr/local/bin/warp COPY server/test/complement/entrypoint.sh /usr/local/bin/warp-complement diff --git a/server/test/complement/README.md b/server/test/complement/README.md index 660f64e..bd28036 100644 --- a/server/test/complement/README.md +++ b/server/test/complement/README.md @@ -6,7 +6,8 @@ Matrix integration suite and this project's primary progress metric drives it over the real HTTP API. ```sh -make complement # the current target set +make complement # the C-S set, federation off +make complement-federation # the federation set bash run.sh TestLogin # one test COMPLEMENT_DEBUG=1 make complement # with the container logs ``` @@ -25,12 +26,26 @@ the phases that are done. ## Profiles -**`federation=off`** — what runs today. Only the Client-Server tests; -`WARP_FEDERATION_ENABLED=false`, port 8448 exposed but unserved. +`COMPLEMENT_PROFILE` picks one; each has its own test set in `run.sh`. -**`federation=on`** — Phase 5. It needs a TLS listener on 8448 with a -certificate signed by the CA that Complement mounts at `/complement/ca`, -which is the first thing to add here when federation lands. +**`csapi`** (the default, `make complement`) — the Client-Server tests in +`tests/csapi`, with `WARP_FEDERATION_ENABLED=false`. Port 8448 is still +served over TLS, because the CA is mounted either way, but the gate answers +every federation request with `M_UNRECOGNIZED`. + +**`federation`** (`make complement-federation`) — the federation tests in +`tests/` and the multi-server tests in `tests/csapi`, with federation on. +The entrypoint has Complement's CA at `/complement/ca` sign a certificate +for the server's name, serves the API on 8448 with it, and points +`federation.tls_ca_file` at the same CA, so that Warp trusts the other +homeservers of the run and the servers Complement itself pretends to be. +The switch reaches the container through `COMPLEMENT_SHARE_ENV_PREFIX`: +`run.sh` sets `PASS_WARP_FEDERATION_ENABLED`, which Complement passes on +as `WARP_FEDERATION_ENABLED`. + +The federation set leaves out partial-state joins (MSC3706, +`TestPartialStateJoin*`), which Warp does not implement, and anything that +needs application services (Phase 9) or device lists (Phase 7). ## The test set diff --git a/server/test/complement/entrypoint.sh b/server/test/complement/entrypoint.sh index a33e0bc..de6656c 100755 --- a/server/test/complement/entrypoint.sh +++ b/server/test/complement/entrypoint.sh @@ -29,11 +29,35 @@ export WARP_RATE_LIMIT_ENABLED=false # Open registration: the tests create their own users. export WARP_REGISTRATION_ENABLED=true -# Federation stays off unless the profile asks for it. Phase 5 adds the -# TLS listener on 8448 that the federation profile needs; until then this -# image only serves the Client-Server API. +# Federation stays off unless the profile asks for it: run.sh passes +# WARP_FEDERATION_ENABLED=true into the container for the federation +# profile. export WARP_FEDERATION_ENABLED="${WARP_FEDERATION_ENABLED:-false}" +# Federation over TLS on 8448. Complement mounts a CA at /complement/ca and +# expects every homeserver of a run to present a certificate it signed; +# the same CA is what Warp trusts when it connects to the others, and to +# the servers Complement itself pretends to be. +CA_DIR=/complement/ca +if [ -f "$CA_DIR/ca.crt" ] && [ -f "$CA_DIR/ca.key" ]; then + if [ ! -f "$DATA_DIR/tls.crt" ]; then + openssl genrsa -out "$DATA_DIR/tls.key" 2048 2>/dev/null + openssl req -new -sha256 -key "$DATA_DIR/tls.key" \ + -subj "/CN=$WARP_SERVER_NAME" -out "$DATA_DIR/tls.csr" + printf 'subjectAltName=DNS:%s\n' "$WARP_SERVER_NAME" > "$DATA_DIR/san.ext" + # The serial file goes in the data directory: the CA mount may be + # read-only. + openssl x509 -req -in "$DATA_DIR/tls.csr" \ + -CA "$CA_DIR/ca.crt" -CAkey "$CA_DIR/ca.key" \ + -CAserial "$DATA_DIR/ca.srl" -CAcreateserial \ + -out "$DATA_DIR/tls.crt" -days 1 -sha256 -extfile "$DATA_DIR/san.ext" 2>/dev/null + fi + export WARP_LISTEN_FEDERATION_ADDRESS=":8448" + export WARP_TLS_CERT_FILE="$DATA_DIR/tls.crt" + export WARP_TLS_KEY_FILE="$DATA_DIR/tls.key" + export WARP_FEDERATION_TLS_CA_FILE="$CA_DIR/ca.crt" +fi + # Complement captures the container's output, so text at debug level is the # most useful thing to have when a test fails. export WARP_LOG_LEVEL="${WARP_LOG_LEVEL:-debug}" diff --git a/server/test/complement/run.sh b/server/test/complement/run.sh index f072711..24ddde3 100755 --- a/server/test/complement/run.sh +++ b/server/test/complement/run.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # Run Complement against a freshly built Warp image. # -# server/test/complement/run.sh # the test set for the phases that are done -# server/test/complement/run.sh TestLogin # one test +# server/test/complement/run.sh # the C-S test set, federation off +# COMPLEMENT_PROFILE=federation server/test/complement/run.sh # the federation test set +# server/test/complement/run.sh TestLogin # one test # COMPLEMENT_TESTS='Test.*' server/test/complement/run.sh # # Complement is a separate Go module, so it has to be checked out and run on @@ -16,17 +17,42 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" : "${COMPLEMENT_REF:=main}" : "${COMPLEMENT_DIR:=${REPO_ROOT}/.complement}" : "${COMPLEMENT_IMAGE:=complement-warp}" +: "${COMPLEMENT_PROFILE:=csapi}" -# The target set for the phases that are done, from PLAN.md section 7. Add -# to it as phases land rather than switching to the whole suite: a pass +# The target sets for the phases that are done, from PLAN.md section 7. Add +# to them as phases land rather than switching to the whole suite: a pass # count that goes down because a later phase is missing is noise. # -# Phase 4 adds the stream tests: /sync in all its forms, /messages, -# /context, filters, typing, receipts and to-device messaging. What is still -# out is anything needing a later phase — the notification counts of the -# push rules (Phase 8), the device lists of the keyserver (Phase 7), and -# the whole of federation (Phase 5). -: "${COMPLEMENT_TESTS:=TestRegistration|TestLogin|TestDeviceManagement|TestRoomCreate|TestRoomState|TestRoomAlias|TestRoomForget|TestMembership.*|TestRedaction.*|TestJoinRoom.*|TestLeaveRoom.*|TestKickRoom.*|TestBanRoom.*|TestPublicRoom.*|TestSync.*|TestMessages.*|TestContext.*|TestFilter.*|TestTyping.*|TestReceipt.*|TestToDevice.*|TestRoomCreationReportsEventsToMyself|TestLazyLoading.*|TestWriteMDirectAccountData}" +# csapi: the Client-Server tests, with federation off. Phase 4 added the +# stream tests: /sync in all its forms, /messages, /context, filters, +# typing, receipts and to-device messaging. What is still out is anything +# needing a later phase — the notification counts of the push rules +# (Phase 8) and the device lists of the keyserver (Phase 7). +CSAPI_TESTS='TestRegistration|TestLogin|TestDeviceManagement|TestRoomCreate|TestRoomState|TestRoomAlias|TestRoomForget|TestMembership.*|TestRedaction.*|TestJoinRoom.*|TestLeaveRoom.*|TestKickRoom.*|TestBanRoom.*|TestPublicRoom.*|TestSync.*|TestMessages.*|TestContext.*|TestFilter.*|TestTyping.*|TestReceipt.*|TestToDevice.*|TestRoomCreationReportsEventsToMyself|TestLazyLoading.*|TestWriteMDirectAccountData' + +# federation: Phase 5. Joins, leaves and invites across servers, keys, +# transactions, missing events, server ACLs and the queries. Partial-state +# joins (MSC3706, TestPartialStateJoin*) are not implemented and are left +# out; so is anything needing an application service (Phase 9) or device +# lists (Phase 7). +FEDERATION_TESTS='TestFederation.*|TestJoinViaRoomIDAndServerName|TestJoinFederatedRoom.*|TestBannedUserCannotSendJoin|TestCannotSendNon.*Via.*|TestCannotSendKnockViaSend.*|TestOutboundFederation.*|TestInboundFederation.*|TestInboundCanReturnMissingEvents|TestRemoteAliasRequestsUnderstandUnicode|TestNetworkPartitionOrdering|TestACLs|TestGetMissingEventsGapFilling|TestEventAuth|TestUnbanViaInvite|TestJoinRoomWithFederation.*|TestRemote.*' + +case "$COMPLEMENT_PROFILE" in + csapi) + : "${COMPLEMENT_TESTS:=$CSAPI_TESTS}" + PACKAGES=(./tests/csapi/...) + FEDERATION_ENABLED=false + ;; + federation) + : "${COMPLEMENT_TESTS:=$FEDERATION_TESTS}" + PACKAGES=(./tests/... ) + FEDERATION_ENABLED=true + ;; + *) + echo "complement: unknown COMPLEMENT_PROFILE '$COMPLEMENT_PROFILE' (want csapi or federation)" >&2 + exit 2 + ;; +esac if [[ $# -gt 0 ]]; then COMPLEMENT_TESTS="$(IFS='|'; echo "$*")" @@ -49,12 +75,18 @@ else git -C "$COMPLEMENT_DIR" checkout -q FETCH_HEAD fi -echo "complement: running '${COMPLEMENT_TESTS}' with federation off" +echo "complement: running '${COMPLEMENT_TESTS}' (profile ${COMPLEMENT_PROFILE}, federation ${FEDERATION_ENABLED})" cd "$COMPLEMENT_DIR" # COMPLEMENT_SPAWN_HS_TIMEOUT_SECS is generous because the first container # of a run has to apply the migrations on a cold SQLite file. +# +# COMPLEMENT_SHARE_ENV_PREFIX hands every host variable starting with +# PASS_ to the homeserver containers, prefix removed: it is how the profile +# reaches the entrypoint's federation switch. COMPLEMENT_BASE_IMAGE="$COMPLEMENT_IMAGE" \ COMPLEMENT_SPAWN_HS_TIMEOUT_SECS="${COMPLEMENT_SPAWN_HS_TIMEOUT_SECS:-60}" \ COMPLEMENT_DEBUG="${COMPLEMENT_DEBUG:-0}" \ - go test -count=1 -v -timeout 20m -run "^(${COMPLEMENT_TESTS})$" ./tests/csapi/... + COMPLEMENT_SHARE_ENV_PREFIX=PASS_ \ + PASS_WARP_FEDERATION_ENABLED="$FEDERATION_ENABLED" \ + go test -count=1 -v -timeout 30m -run "^(${COMPLEMENT_TESTS})$" "${PACKAGES[@]}" diff --git a/warp.yaml.example b/warp.yaml.example index abd7826..95f2ece 100644 --- a/warp.yaml.example +++ b/warp.yaml.example @@ -40,6 +40,15 @@ listen: max_header_bytes: 1048576 # WARP_MAX_HEADER_BYTES shutdown_timeout: 30s # WARP_SHUTDOWN_TIMEOUT + # A second listener serving the same API over TLS, for other servers. + # 8448 is the port they try when nothing delegates them elsewhere. Leave + # it empty behind a reverse proxy that terminates TLS, and point + # global.well_known.server_address at the proxy instead. + # WARP_LISTEN_FEDERATION_ADDRESS + federation_address: "" + tls_cert_file: "" # WARP_TLS_CERT_FILE (PEM chain) + tls_key_file: "" # WARP_TLS_KEY_FILE + database: # sqlite (dev, single user), postgres (production) or pocketbase # (appliance; not implemented yet). WARP_DB_DRIVER @@ -156,6 +165,19 @@ federation: # WARP_FEDERATION_ALLOW_RUNTIME_TOGGLE allow_runtime_toggle: true + # Notaries asked for another server's signing key when that server + # cannot be reached directly, e.g. [matrix.org]. Empty: keys are only + # fetched from their own server. WARP_FEDERATION_TRUSTED_KEY_SERVERS + trusted_key_servers: [] + + # How long other servers may cache this server's key. At most 168h. + # WARP_FEDERATION_KEY_VALIDITY_PERIOD + key_validity_period: 24h + + # Extra CA certificates (PEM) trusted for outbound federation, for a + # private federation with an internal CA. WARP_FEDERATION_TLS_CA_FILE + tls_ca_file: "" + registration: # Off by default. Make the first account with # `warp create-account --username you --admin`, then turn registration