diff --git a/Makefile b/Makefile index 1b2b9a8..37c89ac 100644 --- a/Makefile +++ b/Makefile @@ -96,9 +96,13 @@ complement-image: ## Build the Complement homeserver image docker build -t $(COMPLEMENT_IMAGE) -f $(SERVER_DIR)/test/complement/Dockerfile . .PHONY: complement -complement: ## Run the Complement subset for the phases that are done +complement: ## Run the Complement C-S subset (federation off) bash $(SERVER_DIR)/test/complement/run.sh +.PHONY: complement-federation +complement-federation: ## Run the Complement federation subset + COMPLEMENT_PROFILE=federation bash $(SERVER_DIR)/test/complement/run.sh + .PHONY: docker docker: ## Build the container image docker build -t warp:$(VERSION) . diff --git a/README.md b/README.md index 27e3424..2847701 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,24 @@ what exists today. ## Status +**Phase 5 — federation. Complete.** Warp talks to other homeservers. It +joins rooms on other servers and lets their users join its rooms, invites +across servers with the invitee's server countersigning, rejects remote +invites (locally, if the inviter is gone), and exchanges events, typing, +read receipts and to-device messages with them. Events from elsewhere go +through the spec's full intake — signatures and hashes, the rules against +their own auth events, against the state before them and against the +current state, with missing history fetched and soft failure for what +fails only at the end — and forked rooms are resolved back together. The +outbound queue persists what it owes each server, retries with backoff, +and survives restarts. Server discovery, key fetching with an optional +notary, X-Matrix request signing, server ACLs, and a TLS listener for port +8448. The switch is real now: turning federation off stops the queue and +refuses every outbound request, turning it back on resumes from where it +stopped, and the admin API shows each server's queue and backoff. Known +gaps — no history backfill from other servers, no partial-state joins, no +key rotation — are listed in [docs/federation.md](docs/federation.md). + **Phase 4 — sync. Complete. Milestone M1.** Warp is now a homeserver a client can be used against. `/sync` is real: stream tokens across five independent streams, long polling that wakes the moment something concerns @@ -50,18 +68,18 @@ Client-Server API: | `GET`, `POST /login` · `POST /logout`, `/logout/all` · `POST /refresh` | sessions, with `m.login.password` and refresh tokens | | `GET /account/whoami` · `GET /capabilities` | who am I, and what this build can do | | `GET /devices`, `GET`/`PUT`/`DELETE /devices/{id}` · `POST /delete_devices` | device management; deletion re-asks for the password | -| `GET /profile/{user}` and its `displayname` / `avatar_url` sub-resources | profiles — reads are public, writes are your own | +| `GET /profile/{user}` and its `displayname` / `avatar_url` sub-resources | profiles — reads are public, remote ones asked of their server, writes are your own | | `GET`/`PUT /user/{user}/account_data/{type}` and the per-room variant | account data, stored and returned byte for byte | | `POST /createRoom` | room versions 3 to 12, presets, `initial_state`, invites, aliases | -| `POST /join/{roomIdOrAlias}` · `POST /rooms/{id}/join`, `/leave` | joining by ID or alias, and leaving or rejecting an invite | -| `POST /rooms/{id}/invite`, `/kick`, `/ban`, `/unban` | membership, decided by the room's power levels | +| `POST /join/{roomIdOrAlias}` · `POST /rooms/{id}/join`, `/leave` | joining by ID or alias — on another server too, with federation on — and leaving or rejecting an invite | +| `POST /rooms/{id}/invite`, `/kick`, `/ban`, `/unban` | membership, decided by the room's power levels; invites reach users of other servers | | `PUT /rooms/{id}/send/{type}/{txn}` | messages, idempotent per transaction ID | | `GET`/`PUT /rooms/{id}/state[/{type}[/{key}]]` | room state, read whole or one piece at a time | | `PUT /rooms/{id}/redact/{event}/{txn}` | redactions, applied only when they take effect | | `GET /rooms/{id}/event/{event}` · `/members` (with `?at=`) · `/joined_members` · `/aliases` | reading a room | | `GET /joined_rooms` | the rooms you are in | -| `GET`/`PUT`/`DELETE /directory/room/{alias}` | the alias directory | -| `GET`/`POST /publicRooms` · `GET`/`PUT /directory/list/room/{id}` | the public room directory | +| `GET`/`PUT`/`DELETE /directory/room/{alias}` | the alias directory; remote aliases are asked of their server | +| `GET`/`POST /publicRooms` · `GET`/`PUT /directory/list/room/{id}` | the public room directory, ours or (`?server=`) another server's | | `GET /sync` | the event stream: long-polled, incremental, filtered, with lazy-loaded members | | `GET /rooms/{id}/messages` · `/context/{event}` | paging a room's history, and one event with what surrounds it | | `POST /user/{user}/filter` · `GET /user/{user}/filter/{id}` | filters, stored and returned byte for byte | @@ -71,12 +89,14 @@ Client-Server API: | `GET`/`PUT /presence/{user}/status` | presence — **only while `sync.presence` is on**, and it is off by default | | `GET /_matrix/client/versions` · `/.well-known/matrix/client` | discovery | | `/.well-known/matrix/server` | server discovery — **only while federation is on** | +| `/_matrix/federation/*` · `/_matrix/key/*` | the Server-Server API — **only while federation is on**; see [docs/federation.md](docs/federation.md) | | `GET /healthz` · `GET /metrics` | liveness with a database ping, and Prometheus | | anything else under `/_matrix/` | `404 M_UNRECOGNIZED`, as the spec requires | Admin API under `/_warp/admin/v1`: the federation and registration -switches, user listing and creation, the room list and detail, purging a -room, removing a user from one, and the dashboard counters. See +switches, per-server federation health, user listing and creation, the +room list and detail, purging a room, removing a user from one, and the +dashboard counters. See [docs/admin-api.md](docs/admin-api.md). Also in place: argon2id passwords, hashed access and refresh tokens, @@ -85,11 +105,13 @@ logging, graceful shutdown, the storage interfaces with a working SQLite and PostgreSQL driver (and a PocketBase stub), the embedded migration runner and the runtime settings service. -Complement, the official integration suite, is wired up for the -federation-off profile: `make complement` runs the account, room and stream -tests — `TestRegistration`, `TestLogin`, `TestDeviceManagement`, the Phase 3 -room tests and the Phase 4 `TestSync*`, `TestMessages*`, `TestContext*`, -`TestFilter*`, `TestTyping*`, `TestReceipt*` and `TestToDevice*`. +Complement, the official integration suite, is wired up for both +profiles. `make complement` runs the account, room and stream tests with +federation off — `TestRegistration`, `TestLogin`, `TestDeviceManagement`, +the Phase 3 room tests and the Phase 4 `TestSync*`, `TestMessages*`, +`TestContext*`, `TestFilter*`, `TestTyping*`, `TestReceipt*` and +`TestToDevice*`. `make complement-federation` runs the Phase 5 +Server-Server tests with federation on, over TLS with Complement's CA. ## Quick start @@ -161,17 +183,26 @@ server has ever signed. ## Federation switch Federation is off in a fresh install. While it is off, the federation -surface does not answer at all; `/.well-known/matrix/server` returns -`404 M_UNRECOGNIZED`, exactly as it would on a server that has no -federation API. +surface does not answer at all: `/_matrix/federation/*`, `/_matrix/key/*` +and `/.well-known/matrix/server` return `404 M_UNRECOGNIZED`, exactly as +they would on a server that has no federation API, and nothing outbound +runs. The switch is stored in the `server_settings` table and cached in memory, -so flipping it takes effect on the next request without a restart. In -Phase 0 the only thing behind the gate is the delegation document; the -outbound queue, key fetcher and remote media fetch join it in Phase 5. Set +so flipping it — `PUT /_warp/admin/v1/federation {"enabled": true}` — +takes effect on the next request without a restart. Switching it on +starts the outbound queue, which picks up whatever it still owed other +servers; switching it off stops the queue mid-flight and makes the +federation client refuse every request before it opens a connection. Set `federation.allow_runtime_toggle: false` to pin the switch to the config file. +To federate, other servers have to be able to reach this one: either set +`global.well_known.server_address` to a reverse proxy that terminates TLS, +or let Warp terminate it with `listen.federation_address: ":8448"` and a +certificate. See [docs/configuration.md](docs/configuration.md) and +[docs/federation.md](docs/federation.md). + ## Development ```sh @@ -179,7 +210,8 @@ make test # unit tests (SQLite) make lint # golangci-lint make check # vet + test, what CI runs make cover # coverage.html -make complement # the Complement subset for the phases that are done (needs Docker) +make complement # the Complement C-S subset, federation off (needs Docker) +make complement-federation # the Complement federation subset # the storage tests against PostgreSQL as well WARP_TEST_POSTGRES_DSN='postgres://warp:warp@localhost/warp_test?sslmode=disable' make test-postgres @@ -198,7 +230,8 @@ server/ the homeserver (Go module) logging/ metrics/ slog and Prometheus setup httputil/ Matrix error envelope, CORS, router, middleware ratelimit/ per-endpoint token buckets - federation/ the federation gate and allow/deny policy + federation/ the Server-Server protocol: gate, X-Matrix, discovery, client, keys + federationapi/ federation endpoints, the outbound queue, the switchable Module (docs/federation.md) settings/ runtime settings service (the switches) userapi/ accounts, passwords, devices, tokens, account data roomserver/ rooms: the input pipeline (docs/rooms.md) diff --git a/docs/README.md b/docs/README.md index eafa7cc..d372734 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,6 +7,7 @@ | [event-core.md](event-core.md) | `internal/matrix`: canonical JSON, events, auth rules, state resolution | | [rooms.md](rooms.md) | `internal/roomserver`: the input pipeline, room storage, known gaps | | [sync.md](sync.md) | `internal/syncapi`: stream tokens, long polling, filters, the ephemeral streams | +| [federation.md](federation.md) | the Server-Server API: discovery, keys, taking in events, joins across servers, the outbound queue, the switch | | [admin-api.md](admin-api.md) | `/_warp/admin/v1`: the operator API behind the admin panel | The design and roadmap are in [PLAN.md](../PLAN.md) at the repository root. diff --git a/docs/admin-api.md b/docs/admin-api.md index cfa51b0..d35cafa 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -57,6 +57,43 @@ than offer something that will bounce. Editing the allow and deny lists is Phase 9. They are reported now because the dashboard has to show which servers a running Warp will talk to. +### `GET /federation/destinations` + +Delivery health per remote server: the panel's federation status view. +Every server this one has queued something for, or has failed to reach, +with how much it is still owed. + +```json +{ + "running": true, + "destinations": [ + { + "server_name": "matrix.org", + "failures": 0, + "retry_at_ms": 0, + "last_success_ms": 1760600000000, + "last_failure_ms": 0, + "queued_pdus": 0, + "queued_edus": 0 + } + ] +} +``` + +`running` is whether the outbound queue is up — `false` while federation is +switched off. `failures` counts consecutive failed sends; while it is above +zero the server is being backed off, and `retry_at_ms` is when the next +attempt is due. The backoff doubles from ten seconds to six hours. + +A server started without a signing key has no federation at all, and both +destination endpoints answer `404`. + +### `POST /federation/destinations/{serverName}/reset` + +Forget a server's backoff and try it now — for an operator who knows the +server is back and does not want to wait out a six-hour retry. Answers +`{}`. + ### `GET /registration`, `PUT /registration` The same shape, without the lists, for open registration. diff --git a/docs/configuration.md b/docs/configuration.md index d0e9556..113e05f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -33,6 +33,9 @@ List-valued variables are comma-separated. A single `-` clears a list: | `listen.idle_timeout` | `WARP_IDLE_TIMEOUT` | `120s` | | | `listen.max_header_bytes` | `WARP_MAX_HEADER_BYTES` | `1048576` | | | `listen.shutdown_timeout` | `WARP_SHUTDOWN_TIMEOUT` | `30s` | Grace period for in-flight requests. | +| `listen.federation_address` | `WARP_LISTEN_FEDERATION_ADDRESS` | empty | A second listener serving the same API over TLS, usually `:8448`. Empty = none; see *Federation over TLS*. | +| `listen.tls_cert_file` | `WARP_TLS_CERT_FILE` | empty | PEM certificate chain for the federation listener. Required with it. | +| `listen.tls_key_file` | `WARP_TLS_KEY_FILE` | empty | PEM key for the federation listener. Required with it. | | `database.driver` | `WARP_DB_DRIVER` | `sqlite` | `sqlite`, `postgres`, `pocketbase` (not implemented). | | `database.connection_string` | `WARP_DB_CONNECTION_STRING` | SQLite file in the working directory | | | `database.max_open_conns` | `WARP_DB_MAX_OPEN_CONNS` | `10` | Ignored by SQLite: writes are serialised onto one connection. | @@ -57,6 +60,9 @@ List-valued variables are comma-separated. A single `-` clears a list: | `federation.allowlist` | `WARP_FEDERATION_ALLOWLIST` | `[]` | Non-empty = federate only with these. | | `federation.denylist` | `WARP_FEDERATION_DENYLIST` | `[]` | Always wins over the allowlist. | | `federation.allow_runtime_toggle` | `WARP_FEDERATION_ALLOW_RUNTIME_TOGGLE` | `true` | See below. | +| `federation.trusted_key_servers` | `WARP_FEDERATION_TRUSTED_KEY_SERVERS` | `[]` | Notaries asked for a server's key when it cannot be reached itself, e.g. `matrix.org`. Empty = only ever ask the server. | +| `federation.key_validity_period` | `WARP_FEDERATION_KEY_VALIDITY_PERIOD` | `24h` | How long other servers may cache this server's key. At most `168h`. | +| `federation.tls_ca_file` | `WARP_FEDERATION_TLS_CA_FILE` | empty | Extra CA certificates (PEM) for outbound federation, on top of the system roots. | | `registration.enabled` | `WARP_REGISTRATION_ENABLED` | `false` | | | `registration.allow_runtime_toggle` | `WARP_REGISTRATION_ALLOW_RUNTIME_TOGGLE` | `true` | | | `registration.tokens` | `WARP_REGISTRATION_TOKENS` | `[]` | Non-empty = sign-up also needs one of these, as the `m.login.registration_token` stage. | @@ -145,13 +151,41 @@ Everything else is read once at startup. While `federation.enabled` is false: -- `/.well-known/matrix/server` answers `404 M_UNRECOGNIZED`, and so will - `/_matrix/federation/*` and `/_matrix/key/*` once they exist. -- Nothing outbound runs — in Phase 0 there is nothing outbound yet; from - Phase 5 the transaction queue, key fetcher and remote media fetch stay - stopped. +- `/_matrix/federation/*`, `/_matrix/key/*` and `/.well-known/matrix/server` + answer `404 M_UNRECOGNIZED`, exactly as on a server with no federation + API. The TLS listener on `listen.federation_address`, if there is one, + stays up and answers the same way. +- Nothing outbound runs. The transaction queue is stopped, and the + federation client refuses every request — key fetches, joins, profile + and alias lookups — before it opens a connection, so a component that + somehow missed the switch still cannot reach another server. +- Events persisted while it is off are not queued for other servers. A + remote server that cares about the room fetches what it missed the next + time it hears from this one. What was already queued when the switch + went off stays queued, and goes out when it comes back on. - Client-Server endpoints that would need a remote server answer `M_FORBIDDEN` with a message saying federation is disabled. - Events are still signed with the server key, and room and event IDs are unchanged. A room created while federation is off federates normally once it is switched on. + +## Federation over TLS + +Other servers reach this one at whatever its server name resolves to: the +address in `/.well-known/matrix/server`, an SRV record, or the name itself +on port 8448. Two deployment shapes: + +- **Behind a reverse proxy** that terminates TLS — the usual one. Set + `global.well_known.server_address` to the proxy's `host:443` and leave + `listen.federation_address` empty: the proxy forwards + `/_matrix/federation` and `/_matrix/key` to `listen.address` like + everything else. +- **Warp terminates TLS itself**: set `listen.federation_address` to + `:8448` with `listen.tls_cert_file` and `listen.tls_key_file`. The + certificate must be valid for the server name (or for the delegated + name, if the well-known document delegates). + +Outbound federation verifies certificates against the system roots, plus +`federation.tls_ca_file` when it is set — for a private federation whose +servers share an internal CA. There is no switch to turn verification +off. diff --git a/docs/development.md b/docs/development.md index 07916bd..6d73e9a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -147,15 +147,17 @@ far along the server is — PLAN.md section 9 treats it as the primary progress metric. ```sh -make complement # the current target set +make complement # the C-S set, federation off +make complement-federation # the federation set server/test/complement/run.sh TestLogin # one test COMPLEMENT_DEBUG=1 make complement # with the container logs ``` It needs Docker. The script builds the image, clones Complement into `.complement/` (ignored by git) and runs the subset for the phases that are -done — the account, room and stream tests, with federation off. Extend -`COMPLEMENT_TESTS` in +done. `COMPLEMENT_PROFILE` picks the set: `csapi` (the default) runs the +account, room and stream tests with federation off; `federation` runs the +Server-Server tests with it on. Extend the sets in [run.sh](../server/test/complement/run.sh) as each phase lands, rather than switching to the whole suite: a pass count that drops because a later phase is missing is noise. @@ -165,9 +167,9 @@ and the deployment is entirely `WARP_*` environment variables — see [entrypoint.sh](../server/test/complement/entrypoint.sh). Two things there are worth knowing: rate limiting is off, because the whole suite comes from one address and would otherwise spend its time reading `M_LIMIT_EXCEEDED`; -and port 8448 is exposed but nothing listens on it yet. The federation -profile needs a TLS listener signed by the CA that Complement mounts at -`/complement/ca`, which arrives with federation in Phase 5. +and port 8448 is served over TLS with a certificate the entrypoint has the +CA Complement mounts at `/complement/ca` sign, the same CA Warp is told to +trust for its outbound connections. ## Creating the first account diff --git a/docs/federation.md b/docs/federation.md new file mode 100644 index 0000000..c7d4006 --- /dev/null +++ b/docs/federation.md @@ -0,0 +1,380 @@ +# Federation + +PLAN.md Phase 5: the Server-Server API. Warp can now join rooms on other +servers, have their users join its rooms, exchange events and typing, +receipts and to-device messages with them, and do all of it behind a switch +that turns it off at runtime — cleanly, with nothing left talking to +another server — and back on again, picking up where it left off. + +Federation is still off in a fresh install. Nothing here changes what a +private homeserver does. + +## Where things live + +| Package | What | +|---|---| +| `internal/federation` | The protocol, with no knowledge of rooms: X-Matrix request signing, server discovery, the signed HTTP client, the key ring, the gate and the allow/deny policy. | +| `internal/roomserver` | Rooms across servers: taking in events from elsewhere (`federated.go`), joining, leaving and inviting across servers in both directions (`federated_membership.go`), answering other servers' reads (`federated_query.go`), remote aliases and directories, server ACLs (`acl.go`). | +| `internal/federationapi` | Federation as a running component: the inbound endpoints, the outbound queue, the EDU adapter and the `Module` the switch starts and stops. | + +The split follows the dependencies. The roomserver uses the protocol — it +fetches events and asks for joins — and the component uses the roomserver, +so the component cannot live in the protocol package. PLAN.md calls it +`federation.Module`; it is `federationapi.Module`, for that reason. + +## The switch + +Three things answer to `federation.enabled`, and each reads it at the +moment it matters rather than being told: + +- **The gate.** Every route under `/_matrix/federation`, `/_matrix/key` + and `/.well-known/matrix/server` is wrapped by it, and answers + `404 M_UNRECOGNIZED` while the switch is off — a server with federation + off looks exactly like a server without a federation API. +- **The client.** Every outbound request checks the switch, the allow and + deny lists, and that it is not addressed to this server, before it + resolves anything or opens a connection. A component that somehow missed + the toggle still cannot reach another server. +- **The module.** The one thing that talks to other servers on its own + initiative is the outbound queue, and the module starts and stops it. + The settings service calls the module when the switch moves; stopping + cancels in-flight sends and waits for them, and leaves whatever was not + sent in the database. + +Events persisted while the switch is off are not queued. Holding them would +turn switching federation back on after a month into a month of events +delivered in one burst; a remote server that cares about a room fetches +what it missed with `/get_missing_events` the next time it hears from this +one. What was queued before the switch went off goes out when it comes +back on. + +## Finding other servers + +`federation.Resolver` implements the spec's discovery algorithm, in its +order: an IP literal or an explicit port is used as is; otherwise +`/.well-known/matrix/server` may delegate to another name, which is then +resolved the same way; then the `_matrix-fed._tcp` SRV record, then the +deprecated `_matrix._tcp`, then port 8448. + +The result has two halves and the client keeps them apart. The address to +dial can be an SRV target; the name the TLS certificate must be valid for — +also the `Host` header — is never the SRV target, only the server name or +the name the well-known document delegated to. The client carries the +dial addresses down to its dialer in the request context and builds the URL +from the other half, so Go's TLS verification checks the right name +without any custom verification code. + +Well-known answers are cached for their `Cache-Control` max-age, between +the spec's bounds (24 hours by default, 48 at most); failures for an hour, +so that a server with no document is not asked for one on every request. + +## Keys + +`GET /_matrix/key/v2/server` serves this server's key, valid for +`federation.key_validity_period` (24 hours by default). + +Everybody else's keys go through the key ring. A key response is accepted +only if it is about the server asked for and every signature the server +made on it is by a key it lists and verifies — a response citing a key it +does not publish is broken or forged. Keys are stored in +`federation_server_keys` and cached in memory, and are believed for as long +as the server said, but never for more than a week: the spec's cap. + +When a server's own key endpoint cannot be reached, the notaries in +`federation.trusted_key_servers` are asked, in order; a notary's answer +must carry the notary's signature as well. The default is no notaries: +keys come from the server that owns them or not at all. + +Room versions 5 and later require a key to have been valid when the event +was sent. A cached key that is not known to cover an event's timestamp is +fetched again before the event is refused — the server may have extended +it — and a fetch that fails is not retried for a minute, so that a server +whose key endpoint is down is not asked on every event it sends. + +As a notary, Warp answers for its own keys only. It keeps other servers' +keys, not the signed documents they came in, and a notary has to hand out +the documents. + +## Authenticating requests + +Every federation endpoint but the key and version endpoints checks the +`X-Matrix` Authorization header: the signature covers the method, the +request target exactly as it appeared on the request line, the origin, the +destination and the JSON body. The origin is then checked against the +allow and deny lists, and — for anything about a room — against the room's +server ACL. + +The parser is fuzzed (`FuzzParseXMatrix`): a header that parses must +survive being written back out and parsed again unchanged. + +## Taking in events from other servers + +An event created here is built on the room's current state and can only +fail the authorisation rules. An event from somewhere else arrives with +none of that settled. The roomserver follows the spec's procedure, in +`federated.go`: + +1. **It is what it says it is.** It parses under the room's version, it is + signed by its sender's server, and its content matches its hash. A hash + mismatch with a good signature means somebody redacted it on the way; + the redacted form is kept rather than the event rejected. +2. **It passes the rules against its own auth events.** Missing auth + events are fetched with `/event_auth` and checked themselves, oldest + first. +3. **It passes them against the state before it** — the state after each + of its prev_events, resolved together. A prev_event this server does not + have is asked for with `/get_missing_events` (ten events, between this + server's forward extremities and the new event). A gap wider than that + is closed by asking for the state at the missing event with + `/state_ids`, fetching whatever of it is missing — one at a time with + `/event`, or all at once with `/state` past fifty — and checking all of + it. +4. **It passes them against the current state.** An event that fails only + here is *soft-failed*: kept, because other events may cite it, but never + shown to a client and never a forward extremity. This is what stops a + banned user's server from sending events built on the state before the + ban. + +An event that fails 1 to 3 is rejected and not stored. The `/send` +response says why, per event. + +### Outliers + +Some events have to be held without the room's state at them: the state +and auth chain a join is handed, auth events fetched to judge something +else, the state fetched across a gap, an invite to a room nobody here is +in, a soft-failed event. These are *outliers* — a column on +`roomserver_events` rather than a table of their own, because every read +of a room's current state joins the events table, and the current state of +a room joined over federation is almost all outliers. + +An outlier can be cited and read by ID. It has no state snapshot, is never +a forward extremity, and never appears in `/sync`, `/messages` or +`/context` — the timeline queries filter it out. An outlier that arrives +again as an ordinary event stays an outlier: this server used it as state +without its history, and putting the room's head on it now could put the +head behind events that are already here. + +### The room's head + +A Phase 3 room had one forward extremity, because every event was a child +of the one before. Federation forks rooms: two servers send at the same +moment, each without the other's event. The new extremities are the old +ones the event does not descend from, plus the event. + +The current state then has to be worked out again. In the common case — +the event follows the room's single extremity — it is one tuple more than +before. Otherwise it is resolved across the new extremities and written +whole. The next event created here cites every extremity, and the fork +closes; `TestFederatedJoinAndConversation` checks that both servers end up +with one extremity and the same state. + +### History: backfill + +A room joined over federation starts, on this server, at the join: the +join is handed the room's state, not the events that led to it. When a +client pages `/messages` backwards and the page comes back short at the +bottom of what this server holds, the roomserver asks the servers in the +room for what came before it (`GET /backfill`) before answering, in +`backfill.go`: + +- **Where to start.** The prev_events of the room's oldest timeline events + (the oldest fifty) that are not themselves in the timeline. A room whose + oldest event is its create event has nothing to ask for. +- **Whom to ask.** The servers of the room's members, joined ones first, + in turn, until one sends something that passes. A page asks at most + three times before it is returned short. +- **What is checked.** Steps 1 to 3 of the intake above. The state before + each event comes from the events before it in the same batch, or — + for the oldest ones — from `/state_ids` at its prev_events. Step 4 does + not apply: soft failure refuses events built on a state the room has + moved past, which history always is. An event already held as an + outlier is promoted into the timeline. +- **Where it goes.** Below everything. The stream ordering counter only + goes up, so backfilled events are numbered by a second counter that runs + downwards from -1 (`roomserver_sequences.backfill_ordering`). Each batch + sorts below the one before it, and nothing about the room's head, its + current state or its stream top changes: a live `/sync` reads "after my + token" and never reaches below zero, so history reaches clients only + through `/messages`, `/context` and the state at a point in it. Zero is + never a position, so it can go on meaning "unbounded" in every window; + the position just below the event at 1 is -1 (`storage.PositionBefore`). + +Backfill is only asked for when the user may see before their join — a +room whose history visibility is `joined` or `invited` has nothing to +show them there — and only by `/messages`, never by `/sync` or `/context`, +which do not wait on another server. With federation off, or with nobody +reachable, the page is what this server holds. + +## Membership across servers + +A server can only sign as itself, and a room's state lives with the +servers in it, so each membership change that crosses a server boundary is +a short protocol: + +- **Joining a room this server is not in** — not held at all, held only as + an invite, or held from before every local user left — asks a resident + server for a template (`make_join`), fills it in, signs it, and hands it + back (`send_join`, v2). The answer is the room's state and auth chain, + every event of which is verified and checked against its own auth events + before any of it is believed. Candidates are tried in order: the + client's `server_name` hints, the server in the room ID (before version + 12), the servers of the room's members as last known, and whoever sent + the invite. A restricted join comes back countersigned by the server that + authorised it. +- **Leaving a room this server is not in** — rejecting an invite, in + practice — is the same with `make_leave` and `send_leave`. When no server + in the room can be reached the invite is still rejected locally: a leave + is built and stored here and nowhere else. An invite a user cannot get + rid of because its server is gone is the worse failure. +- **Inviting a user of another server** goes through the ordinary + pipeline, with one extra stage under the room lock: the invite is sent + to the invitee's server to countersign (`/invite`, v2), and the copy with + both signatures is the one persisted. + +And the other side of each: + +- `make_join` builds the template from the room's current state and runs + the rules over a trial build first, so that a banned user hears so before + signing anything. Under the restricted join rule it names a local user to + authorise the join, and `send_join` countersigns it. +- `send_join` and `send_leave` take the event through the same intake as + any other, and mark it to be **relayed**: the joining server is not in + the room yet and relies on this one to tell everybody else. +- `/invite` countersigns an invite to a local user. A room this server is + in takes the invite in as an ordinary event. One it is not in gets it as + an out-of-band membership — an outlier that is nonetheless the invitee's + membership in the room's current state, which is all `/sync` needs — and + the stripped state the inviter sent along is kept in + `roomserver_invite_state` for the invite list, until the invite is + answered. + +## The outbound queue + +`federationapi.Queue` sends this server's events and EDUs to everybody who +should have them. + +The roomserver's emit stage calls it with the room lock held, so that call +only appends to an in-memory list. A dispatcher drains the list, works out +each event's destinations — the servers with a user joined to the room, +plus, for a membership event, the server of the user it is about, so that +a server whose last user was just kicked hears of it — and writes one row +per destination to `federation_queue`. An event received from another +server is not sent on, because its origin sends it to everybody itself; +the exception is a relayed join or leave, which goes to everybody but its +origin. + +A worker per destination sends the rows in transactions of at most 50 PDUs +and 100 EDUs, and deletes them once the destination has answered. A +failure backs the destination off — ten seconds, doubling to six hours, +with a fifth of jitter — in `federation_destinations`, which the admin API +reports and can reset. A destination later denied by the allow and deny +lists has its backlog dropped rather than sent. + +Rows are deleted only once delivered, so a restart loses nothing that had +reached the database: the queue starts a worker for every destination with +a backlog. Transaction IDs carry the queue's start time, because a +receiving server deduplicates by transaction ID and an ID reused after a +restart with different contents would be answered from its cache. + +Received transactions are recorded with their response in +`federation_inbound_txns`, so that a retransmission is answered the same +way rather than processed twice. + +## EDUs + +| EDU | Out | In | +|---|---|---| +| `m.typing` | a local user's typing, to the room's servers | into the typing tracker, if the user belongs to the sending server | +| `m.receipt` | a local user's public read receipts — never private ones | into the receipts stream, likewise | +| `m.direct_to_device` | to-device messages for remote users, one EDU per server | delivered to local devices | +| `m.presence`, `m.device_list_update` | — | ignored: presence is off by default, device lists are Phase 7 | + +Nothing in an EDU is signed, so the one check there is — the user must +belong to the server that sent it — is what keeps a server from making +anybody appear to type. EDUs about a room are subject to its server ACL. + +The sync API hands EDUs over through `syncapi.Outbound`, and only for what +a local user did: the same calls are how EDUs from other servers come in, +and passing those on would echo them back. + +## Server ACLs + +`m.room.server_acl` is enforced against the origin of everything about a +room: its PDUs, its typing and receipts, its joins and leaves, and its +reads of the room's history. Globs match case-insensitively, without the +port; deny wins over allow; an empty allow list allows nobody; and +`allow_ip_literals: false` refuses servers named by an IP address. An ACL +never applies to the server enforcing it. + +## What other servers may read + +`/event`, `/state`, `/state_ids`, `/event_auth`, `/backfill` and +`/get_missing_events` are answered for a server with a user joined to the +room, or for any server if the room is world-readable — and only if the +room's ACL allows it. Anybody else gets `403` whether or not the event +exists. + +`/state_ids` and `/state` answer with the state *before* the event asked +about, which is what every implementation's client side expects; the +asker adds the event itself if it is a state event. An outlier has no +snapshot, and asking for the state at one is a `404`. + +## Endpoints + +| | | +|---|---| +| `GET /_matrix/key/v2/server` · `POST /_matrix/key/v2/query` · `GET /_matrix/key/v2/query/{server}` | keys; the notary answers for this server only | +| `GET /_matrix/federation/v1/version` | | +| `PUT /v1/send/{txnId}` | transactions of PDUs and EDUs | +| `GET /v1/make_join/{room}/{user}` · `PUT /v1/send_join`, `/v2/send_join` | joins | +| `GET /v1/make_leave/{room}/{user}` · `PUT /v1/send_leave`, `/v2/send_leave` | leaves | +| `PUT /v2/invite/{room}/{event}` | invites | +| `GET /v1/event/{event}` · `/v1/state/{room}` · `/v1/state_ids/{room}` · `/v1/event_auth/{room}/{event}` | reads | +| `GET /v1/backfill/{room}` · `POST /v1/get_missing_events/{room}` | history | +| `GET /v1/query/directory` · `/v1/query/profile` | queries | +| `GET`, `POST /v1/publicRooms` | the public room directory | + +Client-Server endpoints that now reach other servers: `/join` and +`/rooms/{id}/join` for a room this server is not in (by ID or by remote +alias), `/invite` for a remote user, `/leave` to reject a remote invite, +`GET /directory/room/{alias}` for a remote alias, `/publicRooms?server=`, +and `GET /profile/{user}` for a remote user. + +## Known gaps + +- **Backfill only fills in history below the room.** A gap higher up — a + live event whose prev_events were closed with `/state_ids` rather than + fetched — stays a gap. And the end of a server's history is not + remembered: a room whose servers cannot supply anything before its + oldest event is asked again each time a client scrolls to the top. +- **Backfilled redactions are not applied** to earlier backfilled events; + a server sends a redacted event in its redacted form anyway, but + `unsigned.redacted_because` is not filled in. +- **No partial-state joins** (MSC3706). Joining a room of tens of + thousands of members means receiving, verifying and checking its whole + state before the join completes, which takes minutes. +- **No key rotation.** The key file holds one key; old keys are not + served as `old_verify_keys`, so rotating the key would invalidate this + server's old events for anybody who has not cached the old key. +- **Rejected events are not remembered.** A rejected event is not stored, + so if it is sent again it is fetched and judged again. +- **Soft-failed events are never re-evaluated**, and a redaction whose + target arrives later is not applied when the target arrives. +- **Room versions 1 and 2** — event IDs carried in the event — are not + supported, so the oldest rooms on the network cannot be joined. +- **Presence and device list EDUs** are ignored on the way in and not sent + (presence is off by default; device lists are Phase 7). +- **Third-party invites** across servers (`exchange_third_party_invite`) + and **knocking** over federation (`make_knock`, `send_knock`) are not + implemented. + +## Testing + +| Where | What | +|---|---| +| `internal/federation` | X-Matrix parsing (fuzzed) and signing, discovery against fake DNS and a fake well-known server, the key ring against a fake key server, an authenticated request end to end over TLS, the client refusing while disabled or denied. | +| `internal/roomserver/federated_test.go` | Two and three roomservers in one process, wired to each other's federation methods: joins at versions 10–12, conversation and fork merging, fail-over, refused joins, invites with countersignatures, rejection with and without the inviter reachable, missing events, a gap closed with state, forged and soft-failed events, ACLs, remote aliases and directories, the switch. `backfill_test.go`: a server that joined late scrolling back through the whole room at versions 10–12, the state inside the history, backfill with the switch off and under `joined` visibility. | +| `internal/federationapi` | The queue: fan-out and relay rules, backoff and retry, transactions within the limits, a restart with a backlog, the policy, EDUs. | +| `internal/setup/phase5_test.go` | Two complete servers over HTTPS: a join by remote alias, messages both ways through the queues, a remote profile, federation switched off with a message held and delivered after it is switched back on, the admin view of it; the TLS listener and the module following the switch. | +| Complement | `make complement-federation` — see [server/test/complement](../server/test/complement/README.md). | diff --git a/docs/rooms.md b/docs/rooms.md index ee206ac..bc0c3f3 100644 --- a/docs/rooms.md +++ b/docs/rooms.md @@ -2,9 +2,9 @@ `internal/roomserver` and the room half of `internal/storage`: PLAN.md Phase 3. This is what turns the event core of Phase 2 into rooms a client -can use — locally. Nothing here talks to another server; federation is -Phase 5, and every operation that would need it is refused with a message -that says so rather than a bare `M_FORBIDDEN`. +can use. Phase 3 built it local only; Phase 5 taught the same pipeline to +take in events from other servers and to join, leave and invite across +them, which [federation.md](federation.md) describes. ## Where the decisions live @@ -65,7 +65,7 @@ With that lock, a Phase 3 room cannot fork, so the resolution path is unreachable from outside. It is implemented anyway because the alternative is a pipeline that authorises against one arbitrary branch of a fork without saying so — and because `stateAfter`, which it needs, is also what -federation will need in Phase 5. +federation needs, and since Phase 5 uses. ### Soft failure @@ -107,7 +107,9 @@ handed out by a counter row inside the write transaction. Neither `AUTOINCREMENT` nor a sequence would do: the two engines spell those differently, and a PostgreSQL sequence is explicitly not transactional, so a rolled-back write would leave a hole that `/sync` would have to be taught to -skip. +skip. History backfilled from other servers is numbered by a second counter +that runs downwards from -1, so that it sorts below everything already +held; see [federation.md](federation.md#history-backfill). **State snapshots** are stored per event and deduplicated by a hash of their contents. That is what makes a per-event snapshot affordable: every message @@ -136,10 +138,10 @@ PLAN.md section 3, enforced here: | Operation | While federation is off | |---|---| -| `/join` a room this server does not have | `403`, naming federation | -| Inviting `@user:other.example` | `403`, naming federation | -| Resolving `#alias:other.example` | `403`, naming federation | -| `publicRooms?server=other.example` | `403`, naming federation | +| `/join` a room this server is not in | `403`, naming federation; with it on, a join through a server that is | +| Inviting `@user:other.example` | `403`, naming federation; with it on, the invite is countersigned by their server | +| Resolving `#alias:other.example` | `403`, naming federation; with it on, the owning server is asked | +| `publicRooms?server=other.example` | `403`, naming federation; with it on, the other server is asked | | Room and event IDs | unchanged — a room created offline federates normally if federation is later turned on | | Signing | unchanged — every event is signed with the server key | @@ -147,10 +149,11 @@ The messages are explicit on purpose. `M_FORBIDDEN` alone sends a user to their operator and the operator hunting through power levels. Restricted joins are the interesting case. The rule needs a signature from -the authorising user's server checked, so it normally needs a key fetcher — -but while federation is off every room is local, this server's own key is in -hand, and the rule can be applied in full. A join authorised via a *remote* -user is refused, like everything else that would need another server. +the authorising user's server checked. This server's own signatures are +checked against the key in hand, so a restricted join authorised by a local +user works whether or not federation is on; one authorised by a remote user +needs the key ring, and while federation is off it is refused like +everything else that would need another server. ## Known gaps diff --git a/docs/sync.md b/docs/sync.md index b13e229..ecd37d8 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -71,6 +71,11 @@ The spec says only that the token is opaque, and clients treat it that way. It is worth being readable anyway: a token is the first thing in every bug report about `/sync`. +The PDU position is the one that can be negative: `/messages` pages back +into history backfilled from other servers, which is numbered downwards +from -1 (see [federation.md](federation.md#history-backfill)), and its +tokens name positions there — `s-12`. The other streams start at zero. + `/messages` and a room's `prev_batch` hand out a PDU-only token. Both are about room events alone, and a token from either is a valid `?since=` for `/sync`, where the other streams then read as unseen — the conservative @@ -112,6 +117,12 @@ type TimelineRequest struct { Low exclusive and High inclusive, **in both directions**. Direction decides only which end the limit trims and which order the store reads in. +Zero is "unbounded" at either end, and is never a position: live events +count up from 1 and backfilled history counts down from -1. So "is this +bound set" is `!= 0`, never `> 0`, and the position just below an event is +`storage.PositionBefore` rather than a subtraction — below 1 is -1, because +a token of zero handed back as a High bound would read the whole room again. + Spelling the bounds as "from" and "to" instead — which is how the Client-Server API spells them — would mean each of them being the low bound in one direction and the high bound in the other, which is exactly the @@ -354,8 +365,8 @@ for a key share is both useless and a leak. A message to a user of another server is dropped rather than refused. A client encrypting for a room with remote members sends one message per device in one request, and failing the request because federation is off -would stop it from delivering to the local devices as well. Phase 5 routes -them. +would stop it from delivering to the local devices as well. With federation +on, they go to the federation queue as `m.direct_to_device` EDUs instead. ### Presence @@ -391,7 +402,7 @@ no timestamp and have to decide what that meant. |---|---|---| | `unread_notifications`, `unread_thread_notifications` | push rules | 8 | | `device_lists.changed` / `.left`, `device_one_time_keys_count` | the keyserver | 7 | -| Remote presence, remote receipts, remote typing | federation | 5 | +| Remote presence | presence over federation | — | | Threads, and threaded receipts beyond storing `thread_id` | threads | 10 | | `contains_url`, `event_fields` | nothing; they are parsed and ignored | — | | Per-event history visibility | state storage (open question 1) | — | diff --git a/server/internal/adminapi/adminapi.go b/server/internal/adminapi/adminapi.go index 8d23bec..7c3e27b 100644 --- a/server/internal/adminapi/adminapi.go +++ b/server/internal/adminapi/adminapi.go @@ -39,6 +39,7 @@ type AdminAPI struct { settings *settings.Service federation *federation.Gate database storage.Database + fedStatus FederationStatus log *slog.Logger } @@ -55,6 +56,10 @@ type Options struct { Settings *settings.Service Federation *federation.Gate + // FederationStatus reports on the federation module. Nil on a server + // without one, and the destinations endpoints then say so. + FederationStatus FederationStatus + // Database is used for the driver name the dashboard shows and for the // migration state; the admin API does not write to it directly. Database storage.Database @@ -85,6 +90,7 @@ func New(opts Options) (*AdminAPI, error) { settings: opts.Settings, federation: opts.Federation, database: opts.Database, + fedStatus: opts.FederationStatus, log: opts.Logger, }, nil } @@ -93,6 +99,8 @@ func New(opts Options) (*AdminAPI, error) { func (a *AdminAPI) RegisterRoutes(r *httputil.Router) { a.handle(r, "GET "+Prefix+"/federation", a.handleGetFederation) a.handle(r, "PUT "+Prefix+"/federation", a.handlePutFederation) + a.handle(r, "GET "+Prefix+"/federation/destinations", a.handleGetDestinations) + a.handle(r, "POST "+Prefix+"/federation/destinations/{serverName}/reset", a.handleResetDestination) a.handle(r, "GET "+Prefix+"/registration", a.handleGetRegistration) a.handle(r, "PUT "+Prefix+"/registration", a.handlePutRegistration) a.handle(r, "GET "+Prefix+"/users", a.handleGetUsers) diff --git a/server/internal/adminapi/federation.go b/server/internal/adminapi/federation.go new file mode 100644 index 0000000..ef8b877 --- /dev/null +++ b/server/internal/adminapi/federation.go @@ -0,0 +1,102 @@ +package adminapi + +import ( + "context" + "net/http" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/httputil" + "github.com/dariuszjaniak/warp/server/internal/storage" + "github.com/dariuszjaniak/warp/server/internal/userapi" +) + +// FederationStatus is what the admin API can see of the running federation +// module: whether it is up, and the delivery health of every server it +// sends to. +type FederationStatus interface { + Running() bool + Destinations(ctx context.Context) ([]storage.DestinationSummary, error) + ResetBackoff(ctx context.Context, serverName string) error +} + +// destinationResponse is one server in the destinations listing. +type destinationResponse struct { + ServerName string `json:"server_name"` + + // Failures is how many sends in a row have failed; zero is healthy. + Failures int `json:"failures"` + + // RetryAtMS is when the next attempt is due, or 0 for now. + RetryAtMS int64 `json:"retry_at_ms"` + LastSuccessMS int64 `json:"last_success_ms"` + LastFailureMS int64 `json:"last_failure_ms"` + LastError string `json:"last_error,omitempty"` + + // QueuedPDUs and QueuedEDUs are the queue depth: what this server + // still owes the destination. + QueuedPDUs int `json:"queued_pdus"` + QueuedEDUs int `json:"queued_edus"` +} + +type destinationsResponse struct { + Running bool `json:"running"` + Destinations []destinationResponse `json:"destinations"` +} + +// requireFederationStatus refuses on a server built without federation. +func (a *AdminAPI) requireFederationStatus() error { + if a.fedStatus == nil { + return httputil.NotFound("This server was started without federation (it has no signing key)") + } + return nil +} + +// handleGetDestinations implements GET /_warp/admin/v1/federation/destinations: +// per-server queue depth and backoff, the panel's federation health view. +func (a *AdminAPI) handleGetDestinations(w http.ResponseWriter, r *http.Request, _ userapi.Authenticated) error { + if err := a.requireFederationStatus(); err != nil { + return err + } + all, err := a.fedStatus.Destinations(r.Context()) + if err != nil { + return err + } + resp := destinationsResponse{ + Running: a.fedStatus.Running(), + Destinations: make([]destinationResponse, 0, len(all)), + } + for _, d := range all { + resp.Destinations = append(resp.Destinations, destinationResponse{ + ServerName: d.ServerName, + Failures: d.Failures, + RetryAtMS: d.RetryAtMS, + LastSuccessMS: d.LastSuccessMS, + LastFailureMS: d.LastFailureMS, + LastError: d.LastError, + QueuedPDUs: d.QueuedPDUs, + QueuedEDUs: d.QueuedEDUs, + }) + } + httputil.NoCache(w) + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} + +// handleResetDestination implements +// POST /_warp/admin/v1/federation/destinations/{serverName}/reset: forget a +// server's backoff and try it now, for an operator who knows it is back. +func (a *AdminAPI) handleResetDestination(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + if err := a.requireFederationStatus(); err != nil { + return err + } + serverName := r.PathValue("serverName") + if _, err := federation.ParseServerName(serverName); err != nil { + return httputil.InvalidParam("%q is not a server name", serverName) + } + if err := a.fedStatus.ResetBackoff(r.Context(), serverName); err != nil { + return err + } + a.log.Info("federation backoff reset through the admin API", "server_name", serverName, "by", auth.UserID) + httputil.WriteJSON(w, http.StatusOK, struct{}{}) + return nil +} diff --git a/server/internal/clientapi/clientapi.go b/server/internal/clientapi/clientapi.go index b30a3ae..2a2e5e3 100644 --- a/server/internal/clientapi/clientapi.go +++ b/server/internal/clientapi/clientapi.go @@ -36,6 +36,7 @@ type ClientAPI struct { sync *syncapi.API settings *settings.Service federation *federation.Gate + remoteProfile RemoteProfiles registration config.Registration log *slog.Logger now func() time.Time @@ -91,6 +92,10 @@ type Options struct { // Federation gates the endpoints that would need a remote server. Federation *federation.Gate + // RemoteProfiles asks another server for one of its users' profile. + // Nil means remote profiles cannot be looked up at all. + RemoteProfiles RemoteProfiles + // Registration is the file configuration of the registration flow. Registration config.Registration @@ -135,6 +140,7 @@ func New(opts Options) (*ClientAPI, error) { sync: opts.Sync, settings: opts.Settings, federation: opts.Federation, + remoteProfile: opts.RemoteProfiles, registration: opts.Registration, log: opts.Logger, now: opts.Now, diff --git a/server/internal/clientapi/profile.go b/server/internal/clientapi/profile.go index 0211eba..5c7ddb1 100644 --- a/server/internal/clientapi/profile.go +++ b/server/internal/clientapi/profile.go @@ -1,11 +1,13 @@ package clientapi import ( + "context" "errors" "net/http" "github.com/dariuszjaniak/warp/server/internal/federation" "github.com/dariuszjaniak/warp/server/internal/httputil" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" "github.com/dariuszjaniak/warp/server/internal/storage" "github.com/dariuszjaniak/warp/server/internal/userapi" ) @@ -59,24 +61,29 @@ func (c *ClientAPI) handleGetAvatarURL(w http.ResponseWriter, r *http.Request) e return nil } -// lookupProfile resolves the user ID in the path to a local account. +// RemoteProfiles asks another server for a profile. *federation.Client +// satisfies it. +type RemoteProfiles interface { + QueryProfile(ctx context.Context, destination, userID, field string) (map[string]any, error) +} + +// lookupProfile resolves the user ID in the path to an account: a local +// one from the database, a remote one by asking its server. // // A user ID on another server is where the federation switch shows up in the // Client-Server API: with federation off there is no way to answer and the // spec's M_FORBIDDEN is the honest reply, as PLAN.md section 3 requires. -// With federation on, the lookup itself is Phase 5 work. func (c *ClientAPI) lookupProfile(r *http.Request) (storage.Account, error) { userID := r.PathValue("userID") localpart, err := c.users.ParseLocalUserID(userID) switch { case errors.Is(err, userapi.ErrForeignServer): - if !c.federation.Enabled() { + if !c.federation.Enabled() || c.remoteProfile == nil { return storage.Account{}, federation.RejectClientRequest( "looking up the profile of a user on another server") } - return storage.Account{}, httputil.NotFound( - "Profiles of users on other servers cannot be fetched yet") + return c.lookupRemoteProfile(r.Context(), userID) case err != nil: return storage.Account{}, httputil.InvalidParam("%s is not a valid user ID", userID) } @@ -155,3 +162,34 @@ func profileWriteError(err error) error { } return err } + +// lookupRemoteProfile asks a user's own server for their profile. +// +// Nothing is cached. A profile changes whenever its owner likes, and a +// cached copy would need invalidating by an EDU this server does not +// receive; the membership events of the rooms a remote user shares with +// ours carry their display name already, which is where clients look for +// it most of the time. +func (c *ClientAPI) lookupRemoteProfile(ctx context.Context, userID string) (storage.Account, error) { + server, err := id.ServerNameOf(userID) + if err != nil { + return storage.Account{}, httputil.InvalidParam("%s is not a valid user ID", userID) + } + profile, err := c.remoteProfile.QueryProfile(ctx, server, userID, "") + switch { + case federation.IsNotFound(err): + return storage.Account{}, httputil.NotFound("Profile not found") + case err != nil: + c.log.Info("remote profile lookup failed", "user_id", userID, "error", err) + return storage.Account{}, httputil.NewError(http.StatusBadGateway, httputil.ErrCodeUnknown, + "%s could not be asked for the profile", server) + } + account := storage.Account{} + if name, ok := profile["displayname"].(string); ok { + account.DisplayName = name + } + if avatar, ok := profile["avatar_url"].(string); ok { + account.AvatarURL = avatar + } + return account, nil +} diff --git a/server/internal/clientapi/profile_test.go b/server/internal/clientapi/profile_test.go index be4f7fc..1fe93aa 100644 --- a/server/internal/clientapi/profile_test.go +++ b/server/internal/clientapi/profile_test.go @@ -1,9 +1,13 @@ package clientapi_test import ( + "context" "net/http" "strings" "testing" + + "github.com/dariuszjaniak/warp/server/internal/clientapi" + "github.com/dariuszjaniak/warp/server/internal/federation" ) const profilePath = "/_matrix/client/v3/profile" @@ -120,7 +124,8 @@ func TestProfileRejectsMalformedUserID(t *testing.T) { // no way to answer, and the spec's M_FORBIDDEN with a clear message is the // honest reply. func TestProfileOfRemoteUserHonoursTheFederationSwitch(t *testing.T) { - ts := newTestServer(t, nil) + remote := fakeRemoteProfiles{"@bob:other.example": {"displayname": "Bob Remote"}} + ts := newTestServer(t, func(o *clientapi.Options) { o.RemoteProfiles = remote }) rec := ts.get(profilePath+"/@bob:other.example", "") expectError(t, rec, http.StatusForbidden, "M_FORBIDDEN") @@ -128,15 +133,33 @@ func TestProfileOfRemoteUserHonoursTheFederationSwitch(t *testing.T) { t.Errorf("the message should say why, got %q", msg) } - // With the switch on it is no longer a refusal, only something that is - // not implemented until Phase 5. + // With the switch on the user's own server is asked. if err := ts.settings.SetFederationEnabled(t.Context(), true); err != nil { t.Fatalf("SetFederationEnabled: %v", err) } - expectError(t, ts.get(profilePath+"/@bob:other.example", ""), + rec = ts.get(profilePath+"/@bob:other.example", "") + if rec.Code != http.StatusOK { + t.Fatalf("remote profile status = %d (body %s)", rec.Code, rec.Body.String()) + } + if name, _ := jsonBody(t, rec)["displayname"].(string); name != "Bob Remote" { + t.Errorf("remote display name = %q", name) + } + expectError(t, ts.get(profilePath+"/@nobody:other.example", ""), http.StatusNotFound, "M_NOT_FOUND") } +// fakeRemoteProfiles answers profile queries from a table, as the remote +// server would. +type fakeRemoteProfiles map[string]map[string]any + +func (f fakeRemoteProfiles) QueryProfile(_ context.Context, _, userID, _ string) (map[string]any, error) { + profile, ok := f[userID] + if !ok { + return nil, &federation.HTTPError{Status: http.StatusNotFound, Code: "M_NOT_FOUND"} + } + return profile, nil +} + func TestProfileRejectsOverlongDisplayName(t *testing.T) { ts := newTestServer(t, nil) acc := ts.register("alice", "superuser") diff --git a/server/internal/clientapi/rooms.go b/server/internal/clientapi/rooms.go index 4d9b254..a9f897e 100644 --- a/server/internal/clientapi/rooms.go +++ b/server/internal/clientapi/rooms.go @@ -46,6 +46,14 @@ func roomError(err error) error { case errors.Is(err, roomserver.ErrFederationDisabled): return httputil.Forbidden("%s", err.Error()) + case errors.Is(err, roomserver.ErrIncompatibleRoomVersion): + return httputil.NewError(http.StatusBadRequest, "M_INCOMPATIBLE_ROOM_VERSION", "%s", err.Error()) + case errors.Is(err, roomserver.ErrNotInRoom): + return httputil.Forbidden("%s", err.Error()) + case errors.Is(err, roomserver.ErrRemote), errors.Is(err, roomserver.ErrBadEvent): + // Another server failed us; the client can do nothing about it but + // try again, or try another server. + return httputil.NewError(http.StatusBadGateway, httputil.ErrCodeUnknown, "%s", err.Error()) case errors.Is(err, roomserver.ErrRejected), errors.Is(err, roomserver.ErrForbidden): return httputil.Forbidden("%s", err.Error()) case errors.Is(err, roomserver.ErrInvalidContent): diff --git a/server/internal/clientapi/sync_test.go b/server/internal/clientapi/sync_test.go index 664e308..60a4303 100644 --- a/server/internal/clientapi/sync_test.go +++ b/server/internal/clientapi/sync_test.go @@ -257,7 +257,10 @@ func TestSyncRejectsAnUnreadableToken(t *testing.T) { ts := newTestServer(t, nil) alice := ts.register("alice", "correct-horse-battery-staple") - for _, since := range []string{"nonsense", "s-1", "sx", "s1_2_3_4_5_6"} { + // "s-1" is not here: a negative event position is a /messages token + // from inside backfilled history, and a valid ?since=. Only the event + // stream goes below zero. + for _, since := range []string{"nonsense", "s1_-1", "sx", "s1_2_3_4_5_6"} { rec := ts.get("/_matrix/client/v3/sync?since="+url.QueryEscape(since), alice.AccessToken) expectError(t, rec, http.StatusBadRequest, "M_INVALID_PARAM") } diff --git a/server/internal/config/config.go b/server/internal/config/config.go index 795c143..3c1e2a5 100644 --- a/server/internal/config/config.go +++ b/server/internal/config/config.go @@ -72,6 +72,20 @@ type Listen struct { // ShutdownTimeout bounds graceful shutdown before connections are cut. ShutdownTimeout time.Duration `yaml:"shutdown_timeout" env:"WARP_SHUTDOWN_TIMEOUT"` + + // FederationAddress, when set, opens a second listener serving the + // same API over TLS. It is the port other servers connect to when + // nothing delegates them elsewhere — 8448 — and the one deployment + // shape where Warp terminates TLS itself rather than sitting behind a + // reverse proxy. Empty means no TLS listener: federation traffic then + // arrives through whatever the well-known document or SRV record + // points at. + FederationAddress string `yaml:"federation_address" env:"WARP_LISTEN_FEDERATION_ADDRESS"` + + // TLSCertFile and TLSKeyFile are the PEM certificate chain and key for + // the federation listener. Both are required when it is enabled. + TLSCertFile string `yaml:"tls_cert_file" env:"WARP_TLS_CERT_FILE"` + TLSKeyFile string `yaml:"tls_key_file" env:"WARP_TLS_KEY_FILE"` } // Database selects and configures the storage driver. @@ -195,6 +209,24 @@ type Federation struct { // AllowRuntimeToggle lets the admin API flip Enabled without a restart. // When true the persisted setting wins over the value above on boot. AllowRuntimeToggle bool `yaml:"allow_runtime_toggle" env:"WARP_FEDERATION_ALLOW_RUNTIME_TOGGLE"` + + // TrustedKeyServers are the notaries asked for another server's + // signing key when that server cannot be reached directly. Empty — + // the default — means keys are only ever fetched from their own + // server, which trusts nobody but requires everybody to be online. + TrustedKeyServers []string `yaml:"trusted_key_servers" env:"WARP_FEDERATION_TRUSTED_KEY_SERVERS"` + + // KeyValidityPeriod is how long this server's key response tells + // others they may cache its key. Shorter means a rotated key is + // noticed sooner; longer means fewer fetches. The spec caps what a + // receiver believes at a week. + KeyValidityPeriod time.Duration `yaml:"key_validity_period" env:"WARP_FEDERATION_KEY_VALIDITY_PERIOD"` + + // TLSCAFile adds a PEM bundle of certificate authorities to the system + // roots for outbound federation, for a private federation whose + // servers have certificates from an internal CA. The system roots + // still apply. + TLSCAFile string `yaml:"tls_ca_file" env:"WARP_FEDERATION_TLS_CA_FILE"` } // Registration controls open registration. @@ -295,6 +327,7 @@ func Default() *Config { Federation: Federation{ Enabled: false, AllowRuntimeToggle: true, + KeyValidityPeriod: 24 * time.Hour, }, Registration: Registration{ Enabled: false, @@ -429,6 +462,17 @@ func (c *Config) Validate() error { bad("federation.denylist[%d]: %w", i, err) } } + for i, s := range c.Federation.TrustedKeyServers { + if err := validateServerName(s); err != nil { + bad("federation.trusted_key_servers[%d]: %w", i, err) + } + } + if c.Federation.KeyValidityPeriod <= 0 || c.Federation.KeyValidityPeriod > 7*24*time.Hour { + bad("federation.key_validity_period: must be positive and at most a week (168h)") + } + if c.Listen.FederationAddress != "" && (c.Listen.TLSCertFile == "" || c.Listen.TLSKeyFile == "") { + bad("listen.federation_address: needs listen.tls_cert_file and listen.tls_key_file") + } return errors.Join(errs...) } diff --git a/server/internal/federation/client.go b/server/internal/federation/client.go new file mode 100644 index 0000000..7a0331d --- /dev/null +++ b/server/internal/federation/client.go @@ -0,0 +1,528 @@ +package federation + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/dariuszjaniak/warp/server/internal/buildinfo" + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" +) + +// Errors an outbound request can fail with before anything is sent. +var ( + // ErrDisabled means federation is switched off. Every outbound request + // checks the switch itself, so that a component that missed the + // toggle still cannot reach another server. + ErrDisabled = errors.New("federation is disabled") + + // ErrNotAllowed means the destination is excluded by the allow and + // deny lists. + ErrNotAllowed = errors.New("federation with that server is not allowed") + + // ErrSelf means a request was addressed to this server. Asking + // ourselves over the network can only be a bug in the caller, and on + // a server whose name does not resolve to itself it would ask + // somebody else. + ErrSelf = errors.New("refusing to send a federation request to this server") +) + +// HTTPError is a non-2xx answer from another server. +type HTTPError struct { + Status int + Code string + Message string +} + +func (e *HTTPError) Error() string { + if e.Code != "" { + return fmt.Sprintf("remote server answered %d %s: %s", e.Status, e.Code, e.Message) + } + return fmt.Sprintf("remote server answered %d", e.Status) +} + +// IsNotFound reports a 404 from another server. +func IsNotFound(err error) bool { + var httpErr *HTTPError + return errors.As(err, &httpErr) && httpErr.Status == http.StatusNotFound +} + +// Response size limits. A send_join answer for a large room is the room's +// entire state and auth chain, which for a public room of tens of +// thousands of members is hundreds of megabytes; nothing else comes close. +const ( + defaultMaxResponse = 16 << 20 + largeMaxResponse = 1 << 30 +) + +// Request timeouts. Joining a large room makes the resident server +// assemble and sign that whole answer first, which takes minutes. +const ( + defaultRequestTimeout = time.Minute + longRequestTimeout = 10 * time.Minute +) + +// ResolveFunc turns a server name into where to connect. +type ResolveFunc func(ctx context.Context, serverName string) (Resolved, error) + +// ClientOptions configure a Client. +type ClientOptions struct { + // ServerName is this server's name: the origin of every request. + ServerName string + + // Key signs every request. + Key *signing.KeyPair + + // Resolve is server discovery. Nil uses a Resolver. + Resolve ResolveFunc + + // TLSConfig is the client TLS configuration, for tests and for + // deployments with a private CA. Nil uses the system roots. + TLSConfig *tls.Config + + // Enabled reports the live federation switch. Nil means off. + Enabled func() bool + + // Policy filters destinations. Nil allows any. + Policy *Policy +} + +// Client sends signed Server-Server API requests. +type Client struct { + serverName string + key *signing.KeyPair + resolve ResolveFunc + http *http.Client + enabled func() bool + policy *Policy + userAgent string +} + +// dialTargetKey carries the resolved addresses of a request down to the +// dialer, which is the only place they are needed: the URL holds the Host +// the certificate is checked against, and the address the connection goes +// to can be something else entirely. +type dialTargetKey struct{} + +// NewClient builds a client. +func NewClient(opts ClientOptions) *Client { + dialer := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second} + tlsConfig := opts.TLSConfig + if tlsConfig == nil { + tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12} + } + transport := &http.Transport{ + Proxy: nil, + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + targets, _ := ctx.Value(dialTargetKey{}).([]string) + if len(targets) == 0 { + return dialer.DialContext(ctx, network, addr) + } + var lastErr error + for _, target := range targets { + conn, err := dialer.DialContext(ctx, network, target) + if err == nil { + return conn, nil + } + lastErr = err + } + return nil, lastErr + }, + TLSClientConfig: tlsConfig, + TLSHandshakeTimeout: 15 * time.Second, + ResponseHeaderTimeout: 0, // bounded by the request context instead + MaxIdleConns: 200, + MaxIdleConnsPerHost: 4, + IdleConnTimeout: 90 * time.Second, + ForceAttemptHTTP2: true, + } + + if opts.Enabled == nil { + opts.Enabled = func() bool { return false } + } + if opts.Policy == nil { + opts.Policy = NewPolicy(nil, nil) + } + c := &Client{ + serverName: opts.ServerName, + key: opts.Key, + resolve: opts.Resolve, + http: &http.Client{ + Transport: transport, + // Federation endpoints do not redirect; following one would + // send a signed request somewhere it was not addressed. + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }, + enabled: opts.Enabled, + policy: opts.Policy, + userAgent: buildinfo.UserAgent(), + } + if c.resolve == nil { + // The well-known document is fetched with ordinary HTTPS and the + // same roots as everything else. + resolver := NewResolver(&http.Client{ + Timeout: 30 * time.Second, + Transport: &http.Transport{TLSClientConfig: tlsConfig, Proxy: nil}, + }) + c.resolve = resolver.Resolve + } + return c +} + +// ServerName returns the origin this client signs as. +func (c *Client) ServerName() string { return c.serverName } + +// check refuses a destination before anything goes on the wire. +func (c *Client) check(destination string) error { + switch { + case !c.enabled(): + return ErrDisabled + case destination == c.serverName: + return ErrSelf + case !c.policy.Allowed(destination): + return fmt.Errorf("%w: %s", ErrNotAllowed, destination) + } + return nil +} + +// request describes one call. +type request struct { + method string + destination string + path string // already escaped + query url.Values + body any + signed bool + maxBytes int64 + timeout time.Duration +} + +// do performs a request and decodes a JSON answer into out (which may be +// nil). +func (c *Client) do(ctx context.Context, req request, out any) error { + if err := c.check(req.destination); err != nil { + return err + } + resolved, err := c.resolve(ctx, req.destination) + if err != nil { + return fmt.Errorf("resolve %s: %w", req.destination, err) + } + + timeout := req.timeout + if timeout == 0 { + timeout = defaultRequestTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + ctx = context.WithValue(ctx, dialTargetKey{}, resolved.Addresses) + + var content []byte + if req.body != nil { + if content, err = json.Marshal(req.body); err != nil { + return fmt.Errorf("encode request: %w", err) + } + } + + target := "https://" + resolved.Host + req.path + if len(req.query) > 0 { + target += "?" + req.query.Encode() + } + httpReq, err := http.NewRequestWithContext(ctx, req.method, target, bytes.NewReader(content)) + if err != nil { + return fmt.Errorf("build request: %w", err) + } + if content == nil { + httpReq.Body = http.NoBody + } else { + httpReq.Header.Set("Content-Type", "application/json") + } + httpReq.Header.Set("User-Agent", c.userAgent) + httpReq.Header.Set("Accept", "application/json") + + if req.signed { + // The signature covers the request target exactly as it goes on + // the wire, so it is taken from the built request rather than + // from the path this function was handed. + x, err := SignRequest(c.key, c.serverName, req.destination, req.method, + httpReq.URL.RequestURI(), content) + if err != nil { + return fmt.Errorf("sign request: %w", err) + } + httpReq.Header.Set("Authorization", x.String()) + } + + resp, err := c.http.Do(httpReq) + if err != nil { + return fmt.Errorf("%s %s%s: %w", req.method, req.destination, req.path, err) + } + defer func() { _ = resp.Body.Close() }() + + limit := req.maxBytes + if limit == 0 { + limit = defaultMaxResponse + } + body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1)) + if err != nil { + return fmt.Errorf("%s %s%s: read response: %w", req.method, req.destination, req.path, err) + } + if int64(len(body)) > limit { + return fmt.Errorf("%s %s%s: response is larger than %d bytes", req.method, req.destination, req.path, limit) + } + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + httpErr := &HTTPError{Status: resp.StatusCode} + var envelope struct { + Code string `json:"errcode"` + Message string `json:"error"` + } + if json.Unmarshal(body, &envelope) == nil { + httpErr.Code, httpErr.Message = envelope.Code, envelope.Message + } + return httpErr + } + if out == nil { + return nil + } + if err := json.Unmarshal(body, out); err != nil { + return fmt.Errorf("%s %s%s: decode response: %w", req.method, req.destination, req.path, err) + } + return nil +} + +// esc escapes one path segment. Room, event and user IDs carry sigils and +// colons, which are legal in a path; slashes would not be, and an event ID +// of room versions 3 and before can contain one. +func esc(s string) string { return url.PathEscape(s) } + +// ServerKeys fetches a server's own key response, unsigned and unverified: +// verifying it is the key ring's job. +func (c *Client) ServerKeys(ctx context.Context, serverName string) (json.RawMessage, error) { + var out json.RawMessage + err := c.do(ctx, request{method: http.MethodGet, destination: serverName, path: "/_matrix/key/v2/server"}, &out) + return out, err +} + +// QueryKeys asks a notary for other servers' keys. +func (c *Client) QueryKeys(ctx context.Context, notary string, query map[string]map[string]KeyCriteria) ([]json.RawMessage, error) { + var out struct { + ServerKeys []json.RawMessage `json:"server_keys"` + } + err := c.do(ctx, request{ + method: http.MethodPost, destination: notary, path: "/_matrix/key/v2/query", + body: map[string]any{"server_keys": query}, + }, &out) + return out.ServerKeys, err +} + +// KeyCriteria is one key asked of a notary. +type KeyCriteria struct { + MinimumValidUntilTS int64 `json:"minimum_valid_until_ts,omitempty"` +} + +// SendTransaction delivers a transaction. +func (c *Client) SendTransaction(ctx context.Context, destination, txnID string, txn Transaction) (SendTransactionResponse, error) { + var out SendTransactionResponse + err := c.do(ctx, request{ + method: http.MethodPut, destination: destination, signed: true, + path: "/_matrix/federation/v1/send/" + esc(txnID), body: txn, + }, &out) + return out, err +} + +// MakeJoin asks a resident server for a join event template. +func (c *Client) MakeJoin(ctx context.Context, destination, roomID, userID string, versions []string) (MakeJoinResponse, error) { + query := url.Values{} + for _, v := range versions { + query.Add("ver", v) + } + var out MakeJoinResponse + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/make_join/" + esc(roomID) + "/" + esc(userID), + query: query, + }, &out) + return out, err +} + +// SendJoin submits a signed join and receives the room's state. +func (c *Client) SendJoin(ctx context.Context, destination, roomID, eventID string, joinEvent json.RawMessage) (SendJoinResponse, error) { + var out SendJoinResponse + err := c.do(ctx, request{ + method: http.MethodPut, destination: destination, signed: true, + path: "/_matrix/federation/v2/send_join/" + esc(roomID) + "/" + esc(eventID), + body: joinEvent, + maxBytes: largeMaxResponse, + timeout: longRequestTimeout, + }, &out) + return out, err +} + +// MakeLeave asks a resident server for a leave event template. +func (c *Client) MakeLeave(ctx context.Context, destination, roomID, userID string) (MakeJoinResponse, error) { + var out MakeJoinResponse + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/make_leave/" + esc(roomID) + "/" + esc(userID), + }, &out) + return out, err +} + +// SendLeave submits a signed leave. +func (c *Client) SendLeave(ctx context.Context, destination, roomID, eventID string, leaveEvent json.RawMessage) error { + return c.do(ctx, request{ + method: http.MethodPut, destination: destination, signed: true, + path: "/_matrix/federation/v2/send_leave/" + esc(roomID) + "/" + esc(eventID), + body: leaveEvent, + }, nil) +} + +// Invite asks the invitee's server to countersign an invite. +func (c *Client) Invite(ctx context.Context, destination, roomID, eventID string, req InviteRequest) (InviteResponse, error) { + var out InviteResponse + err := c.do(ctx, request{ + method: http.MethodPut, destination: destination, signed: true, + path: "/_matrix/federation/v2/invite/" + esc(roomID) + "/" + esc(eventID), + body: req, + }, &out) + return out, err +} + +// Event fetches one event. +func (c *Client) Event(ctx context.Context, destination, eventID string) (Transaction, error) { + var out Transaction + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/event/" + esc(eventID), + }, &out) + return out, err +} + +// EventAuth fetches the auth chain of an event. +func (c *Client) EventAuth(ctx context.Context, destination, roomID, eventID string) (EventAuthResponse, error) { + var out EventAuthResponse + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/event_auth/" + esc(roomID) + "/" + esc(eventID), + maxBytes: largeMaxResponse, + timeout: longRequestTimeout, + }, &out) + return out, err +} + +// StateIDs fetches the IDs of the room state at an event. +func (c *Client) StateIDs(ctx context.Context, destination, roomID, eventID string) (StateIDsResponse, error) { + var out StateIDsResponse + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/state_ids/" + esc(roomID), + query: url.Values{"event_id": {eventID}}, + maxBytes: largeMaxResponse, + timeout: longRequestTimeout, + }, &out) + return out, err +} + +// State fetches the room state at an event, events and all. +func (c *Client) State(ctx context.Context, destination, roomID, eventID string) (StateResponse, error) { + var out StateResponse + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/state/" + esc(roomID), + query: url.Values{"event_id": {eventID}}, + maxBytes: largeMaxResponse, + timeout: longRequestTimeout, + }, &out) + return out, err +} + +// MissingEvents asks for the events between what this server has and an +// event it was sent. +func (c *Client) MissingEvents(ctx context.Context, destination, roomID string, req MissingEventsRequest) (MissingEventsResponse, error) { + var out MissingEventsResponse + err := c.do(ctx, request{ + method: http.MethodPost, destination: destination, signed: true, + path: "/_matrix/federation/v1/get_missing_events/" + esc(roomID), + body: req, + }, &out) + return out, err +} + +// Backfill asks for history before the given events. +func (c *Client) Backfill(ctx context.Context, destination, roomID string, from []string, limit int) (Transaction, error) { + query := url.Values{"limit": {strconv.Itoa(limit)}} + for _, id := range from { + query.Add("v", id) + } + var out Transaction + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/backfill/" + esc(roomID), + query: query, + }, &out) + return out, err +} + +// QueryProfile asks a server for one of its users' profile. An empty field +// asks for the whole profile. +func (c *Client) QueryProfile(ctx context.Context, destination, userID, field string) (map[string]any, error) { + query := url.Values{"user_id": {userID}} + if field != "" { + query.Set("field", field) + } + var out map[string]any + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/query/profile", query: query, + }, &out) + return out, err +} + +// QueryDirectory resolves an alias on the server that owns it. +func (c *Client) QueryDirectory(ctx context.Context, destination, alias string) (DirectoryResponse, error) { + var out DirectoryResponse + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/query/directory", query: url.Values{"room_alias": {alias}}, + }, &out) + return out, err +} + +// PublicRooms reads another server's public room directory. +func (c *Client) PublicRooms(ctx context.Context, destination string, limit int, since, search string) (json.RawMessage, error) { + body := map[string]any{} + if limit > 0 { + body["limit"] = limit + } + if since != "" { + body["since"] = since + } + if search = strings.TrimSpace(search); search != "" { + body["filter"] = map[string]string{"generic_search_term": search} + } + var out json.RawMessage + err := c.do(ctx, request{ + method: http.MethodPost, destination: destination, signed: true, + path: "/_matrix/federation/v1/publicRooms", body: body, + }, &out) + return out, err +} + +// Version asks a server what software it runs. +func (c *Client) Version(ctx context.Context, destination string) (VersionResponse, error) { + var out VersionResponse + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, + path: "/_matrix/federation/v1/version", + }, &out) + return out, err +} diff --git a/server/internal/federation/discovery.go b/server/internal/federation/discovery.go new file mode 100644 index 0000000..c21360e --- /dev/null +++ b/server/internal/federation/discovery.go @@ -0,0 +1,322 @@ +package federation + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "strconv" + "strings" + "sync" + "time" +) + +// Server discovery: turning a server name into somewhere to connect. +// +// A Matrix server name is not an address. "example.org" may be served from +// matrix.example.org:443 because example.org says so in a well-known +// document, or from whatever an SRV record names, or from example.org:8448 +// when neither exists. The spec's algorithm decides, in a fixed order, and +// the result has two halves that must not be confused: the address to dial, +// and the name the TLS certificate must be valid for (which is also the +// Host header). An SRV target is only ever the first: delegating by SRV +// does not let the target's certificate stand in for the server's. + +// DefaultFederationPort is the port a server name without one is served on +// when nothing delegates it elsewhere. +const DefaultFederationPort = 8448 + +// ErrInvalidServerName is a server name that does not parse. +var ErrInvalidServerName = errors.New("invalid server name") + +// ServerName is a parsed server name: a host, which is a DNS name or an IP +// literal, and an optional port. +type ServerName struct { + Host string + Port int + + // IsIP reports that Host is an IP literal, without the brackets an + // IPv6 literal is written with. + IsIP bool +} + +// ParseServerName parses "host", "host:port", "1.2.3.4:port" or +// "[::1]:port". +func ParseServerName(s string) (ServerName, error) { + if s == "" || len(s) > 255 { + return ServerName{}, fmt.Errorf("%w: %q", ErrInvalidServerName, s) + } + + host, portStr := s, "" + if strings.HasPrefix(s, "[") { + end := strings.IndexByte(s, ']') + if end < 0 { + return ServerName{}, fmt.Errorf("%w: %q has an unterminated IPv6 literal", ErrInvalidServerName, s) + } + host = s[1:end] + rest := s[end+1:] + if rest != "" { + if !strings.HasPrefix(rest, ":") { + return ServerName{}, fmt.Errorf("%w: %q", ErrInvalidServerName, s) + } + portStr = rest[1:] + } + ip := net.ParseIP(host) + if ip == nil || ip.To4() != nil { + return ServerName{}, fmt.Errorf("%w: %q is not an IPv6 literal", ErrInvalidServerName, s) + } + } else if i := strings.LastIndexByte(s, ':'); i >= 0 { + host, portStr = s[:i], s[i+1:] + } + + name := ServerName{Host: host} + if portStr != "" || strings.HasSuffix(s, ":") { + port, err := strconv.Atoi(portStr) + if err != nil || port < 1 || port > 65535 { + return ServerName{}, fmt.Errorf("%w: %q has a bad port", ErrInvalidServerName, s) + } + name.Port = port + } + + if ip := net.ParseIP(host); ip != nil { + if ip.To4() == nil && !strings.HasPrefix(s, "[") { + // A bare IPv6 address with a port is ambiguous, which is why + // the grammar requires the brackets. + return ServerName{}, fmt.Errorf("%w: %q: an IPv6 literal must be bracketed", ErrInvalidServerName, s) + } + name.IsIP = true + return name, nil + } + if !validDNSName(host) { + return ServerName{}, fmt.Errorf("%w: %q is not a DNS name", ErrInvalidServerName, s) + } + return name, nil +} + +// validDNSName applies the spec's dns-name grammar: letters, digits, '-' +// and '.', at most 255 characters. +func validDNSName(s string) bool { + if s == "" || len(s) > 255 { + return false + } + for _, r := range s { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '.': + default: + return false + } + } + return true +} + +// hostPort renders a host and port for dialling, bracketing IPv6. +func hostPort(host string, port int) string { + return net.JoinHostPort(host, strconv.Itoa(port)) +} + +// Resolved is where to send a server's federation traffic. +type Resolved struct { + // Addresses are host:port pairs to try in order. + Addresses []string + + // Host is the Host header and, without its port, the name the TLS + // certificate must be valid for. + Host string +} + +// TLSServerName is the name the server's certificate must match. +func (r Resolved) TLSServerName() string { + if host, _, err := net.SplitHostPort(r.Host); err == nil { + return host + } + return strings.Trim(r.Host, "[]") +} + +// Resolver implements server discovery, caching well-known lookups. +type Resolver struct { + // HTTPClient fetches /.well-known/matrix/server. It should verify TLS + // normally: the well-known document is fetched from the server name + // itself, over ordinary HTTPS. + HTTPClient *http.Client + + // LookupSRV is net.DefaultResolver.LookupSRV unless a test replaces it. + LookupSRV func(ctx context.Context, service, proto, name string) (string, []*net.SRV, error) + + Now func() time.Time + + mu sync.Mutex + cache map[string]wellKnownEntry +} + +type wellKnownEntry struct { + server string // the delegated m.server, or "" when there is none + expires time.Time +} + +// Well-known cache lifetimes, from the spec's guidance: a successful +// lookup is cached for what Cache-Control says, within these bounds, and a +// failed one for an hour, so that a server with no well-known document is +// not asked for one on every request. +const ( + wellKnownDefaultTTL = 24 * time.Hour + wellKnownMaxTTL = 48 * time.Hour + wellKnownErrorTTL = time.Hour + + // wellKnownMaxBytes caps the document. A real one is a line of JSON. + wellKnownMaxBytes = 64 << 10 +) + +// NewResolver builds a resolver around an HTTP client. +func NewResolver(client *http.Client) *Resolver { + if client == nil { + client = &http.Client{Timeout: 30 * time.Second} + } + return &Resolver{ + HTTPClient: client, + LookupSRV: net.DefaultResolver.LookupSRV, + Now: time.Now, + cache: map[string]wellKnownEntry{}, + } +} + +// Resolve runs the discovery algorithm for a server name. +func (r *Resolver) Resolve(ctx context.Context, serverName string) (Resolved, error) { + name, err := ParseServerName(serverName) + if err != nil { + return Resolved{}, err + } + + // Step 1: an IP literal is its own address. + if name.IsIP { + port := name.Port + if port == 0 { + port = DefaultFederationPort + } + return Resolved{Addresses: []string{hostPort(name.Host, port)}, Host: serverName}, nil + } + // Step 2: an explicit port means no delegation of any kind. + if name.Port != 0 { + return Resolved{Addresses: []string{hostPort(name.Host, name.Port)}, Host: serverName}, nil + } + + // Step 3: a well-known document can delegate to another name. + if delegated := r.wellKnown(ctx, name.Host); delegated != "" { + dname, err := ParseServerName(delegated) + if err == nil { + switch { + case dname.IsIP: + port := dname.Port + if port == 0 { + port = DefaultFederationPort + } + return Resolved{Addresses: []string{hostPort(dname.Host, port)}, Host: delegated}, nil + case dname.Port != 0: + return Resolved{Addresses: []string{hostPort(dname.Host, dname.Port)}, Host: delegated}, nil + default: + return r.srvOrDefault(ctx, dname.Host), nil + } + } + // A malformed delegation is treated as no delegation, which is + // what the spec asks for an invalid well-known response. + } + + // Steps 4 to 6: SRV records, then the default port. + return r.srvOrDefault(ctx, name.Host), nil +} + +// srvOrDefault looks up the SRV records of a hostname, the current service +// name first and the deprecated one second, falling back to port 8448. +// The Host header stays the hostname whatever the SRV target is. +func (r *Resolver) srvOrDefault(ctx context.Context, host string) Resolved { + for _, service := range []string{"matrix-fed", "matrix"} { + _, records, err := r.LookupSRV(ctx, service, "tcp", host) + if err != nil || len(records) == 0 { + continue + } + var addrs []string + for _, rec := range records { + target := strings.TrimSuffix(rec.Target, ".") + if target == "" { + continue + } + addrs = append(addrs, hostPort(target, int(rec.Port))) + } + if len(addrs) > 0 { + return Resolved{Addresses: addrs, Host: host} + } + } + return Resolved{Addresses: []string{hostPort(host, DefaultFederationPort)}, Host: host} +} + +// wellKnown returns the delegated server name for a host, or "". +func (r *Resolver) wellKnown(ctx context.Context, host string) string { + now := r.Now() + r.mu.Lock() + if entry, ok := r.cache[host]; ok && now.Before(entry.expires) { + r.mu.Unlock() + return entry.server + } + r.mu.Unlock() + + server, ttl := r.fetchWellKnown(ctx, host) + + r.mu.Lock() + r.cache[host] = wellKnownEntry{server: server, expires: now.Add(ttl)} + r.mu.Unlock() + return server +} + +// fetchWellKnown fetches and parses the document, returning the +// delegation and how long to believe it. +func (r *Resolver) fetchWellKnown(ctx context.Context, host string) (string, time.Duration) { + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+host+"/.well-known/matrix/server", nil) + if err != nil { + return "", wellKnownErrorTTL + } + resp, err := r.HTTPClient.Do(req) + if err != nil { + return "", wellKnownErrorTTL + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return "", wellKnownErrorTTL + } + body, err := io.ReadAll(io.LimitReader(resp.Body, wellKnownMaxBytes)) + if err != nil { + return "", wellKnownErrorTTL + } + var doc struct { + Server string `json:"m.server"` + } + if err := json.Unmarshal(body, &doc); err != nil || doc.Server == "" { + return "", wellKnownErrorTTL + } + return doc.Server, cacheTTL(resp.Header.Get("Cache-Control")) +} + +// cacheTTL reads max-age from a Cache-Control header, within the bounds +// the spec sets for a well-known lookup. +func cacheTTL(header string) time.Duration { + for _, directive := range strings.Split(header, ",") { + name, value, ok := strings.Cut(strings.TrimSpace(directive), "=") + if !ok || !strings.EqualFold(name, "max-age") { + continue + } + seconds, err := strconv.Atoi(strings.Trim(value, `"`)) + if err != nil || seconds < 0 { + break + } + ttl := time.Duration(seconds) * time.Second + if ttl > wellKnownMaxTTL { + ttl = wellKnownMaxTTL + } + return ttl + } + return wellKnownDefaultTTL +} diff --git a/server/internal/federation/discovery_test.go b/server/internal/federation/discovery_test.go new file mode 100644 index 0000000..adb8566 --- /dev/null +++ b/server/internal/federation/discovery_test.go @@ -0,0 +1,185 @@ +package federation + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestParseServerName(t *testing.T) { + cases := []struct { + in string + want ServerName + bad bool + }{ + {in: "example.org", want: ServerName{Host: "example.org"}}, + {in: "example.org:8448", want: ServerName{Host: "example.org", Port: 8448}}, + {in: "1.2.3.4", want: ServerName{Host: "1.2.3.4", IsIP: true}}, + {in: "1.2.3.4:80", want: ServerName{Host: "1.2.3.4", Port: 80, IsIP: true}}, + {in: "[::1]", want: ServerName{Host: "::1", IsIP: true}}, + {in: "[2001:db8::1]:8448", want: ServerName{Host: "2001:db8::1", Port: 8448, IsIP: true}}, + {in: "", bad: true}, + {in: "::1", bad: true}, + {in: "example.org:", bad: true}, + {in: "example.org:0", bad: true}, + {in: "example.org:70000", bad: true}, + {in: "exa mple.org", bad: true}, + {in: "[1.2.3.4]", bad: true}, + {in: "[::1", bad: true}, + } + for _, tc := range cases { + got, err := ParseServerName(tc.in) + if tc.bad { + if !errors.Is(err, ErrInvalidServerName) { + t.Errorf("ParseServerName(%q) = %+v, %v; want an error", tc.in, got, err) + } + continue + } + if err != nil || got != tc.want { + t.Errorf("ParseServerName(%q) = %+v, %v; want %+v", tc.in, got, err, tc.want) + } + } +} + +// fakeDNS answers SRV lookups from a table. +type fakeDNS map[string][]*net.SRV + +func (f fakeDNS) lookup(_ context.Context, service, proto, name string) (string, []*net.SRV, error) { + records, ok := f["_"+service+"._"+proto+"."+name] + if !ok { + return "", nil, &net.DNSError{Err: "no such host", Name: name, IsNotFound: true} + } + return "", records, nil +} + +// wellKnownServer serves /.well-known/matrix/server for any Host, from a +// table, counting requests. +func wellKnownServer(t *testing.T, docs map[string]string, hits *atomic.Int32) *Resolver { + t.Helper() + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + doc, ok := docs[strings.Split(r.Host, ":")[0]] + if !ok || r.URL.Path != "/.well-known/matrix/server" { + http.NotFound(w, r) + return + } + w.Header().Set("Cache-Control", "max-age=60") + _, _ = w.Write([]byte(doc)) + })) + t.Cleanup(srv.Close) + + // Every hostname is dialled at the test server, whose certificate the + // client trusts, so the well-known fetch is real HTTPS to a fake name. + transport := srv.Client().Transport.(*http.Transport).Clone() + addr := srv.Listener.Addr().String() + transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, addr) + } + transport.TLSClientConfig.InsecureSkipVerify = true //nolint:gosec // test server with a fixed certificate + r := NewResolver(&http.Client{Transport: transport}) + return r +} + +func TestResolve(t *testing.T) { + var hits atomic.Int32 + r := wellKnownServer(t, map[string]string{ + "delegated.example": `{"m.server":"matrix.delegated.example"}`, + "withport.example": `{"m.server":"matrix.withport.example:443"}`, + "toip.example": `{"m.server":"10.0.0.1"}`, + "broken.example": `{"m.server":"not a name"}`, + }, &hits) + r.LookupSRV = fakeDNS{ + "_matrix-fed._tcp.matrix.delegated.example": {{Target: "fed.delegated.example.", Port: 8443}}, + "_matrix-fed._tcp.srv.example": {{Target: "a.srv.example.", Port: 1}, {Target: "b.srv.example.", Port: 2}}, + "_matrix._tcp.legacy.example": {{Target: "old.legacy.example.", Port: 9}}, + }.lookup + + cases := []struct { + name string + addresses []string + host string + }{ + {"1.2.3.4", []string{"1.2.3.4:8448"}, "1.2.3.4"}, + {"[::1]:9000", []string{"[::1]:9000"}, "[::1]:9000"}, + {"explicit.example:1234", []string{"explicit.example:1234"}, "explicit.example:1234"}, + // Delegation by well-known, then SRV on the delegated name: the + // SRV target is dialled, the delegated name is what TLS checks. + {"delegated.example", []string{"fed.delegated.example:8443"}, "matrix.delegated.example"}, + {"withport.example", []string{"matrix.withport.example:443"}, "matrix.withport.example:443"}, + {"toip.example", []string{"10.0.0.1:8448"}, "10.0.0.1"}, + {"broken.example", []string{"broken.example:8448"}, "broken.example"}, + {"srv.example", []string{"a.srv.example:1", "b.srv.example:2"}, "srv.example"}, + {"legacy.example", []string{"old.legacy.example:9"}, "legacy.example"}, + {"plain.example", []string{"plain.example:8448"}, "plain.example"}, + } + for _, tc := range cases { + got, err := r.Resolve(t.Context(), tc.name) + if err != nil { + t.Errorf("Resolve(%s): %v", tc.name, err) + continue + } + if !slices.Equal(got.Addresses, tc.addresses) || got.Host != tc.host { + t.Errorf("Resolve(%s) = %+v, want %v via Host %s", tc.name, got, tc.addresses, tc.host) + } + } + + if got, _ := r.Resolve(t.Context(), "delegated.example"); got.TLSServerName() != "matrix.delegated.example" { + t.Errorf("TLSServerName = %q", got.TLSServerName()) + } + if got, _ := r.Resolve(t.Context(), "[::1]:9000"); got.TLSServerName() != "::1" { + t.Errorf("TLSServerName of an IPv6 literal = %q", got.TLSServerName()) + } +} + +func TestWellKnownIsCached(t *testing.T) { + var hits atomic.Int32 + r := wellKnownServer(t, map[string]string{"cached.example": `{"m.server":"other.example:443"}`}, &hits) + r.LookupSRV = fakeDNS{}.lookup + now := time.Unix(1_700_000_000, 0) + r.Now = func() time.Time { return now } + + for range 3 { + if _, err := r.Resolve(t.Context(), "cached.example"); err != nil { + t.Fatal(err) + } + } + // A server with no document is cached too, as a failure. + for range 3 { + if _, err := r.Resolve(t.Context(), "missing.example"); err != nil { + t.Fatal(err) + } + } + if n := hits.Load(); n != 2 { + t.Errorf("%d well-known fetches, want one per name", n) + } + + now = now.Add(2 * time.Minute) // past max-age=60 + if _, err := r.Resolve(t.Context(), "cached.example"); err != nil { + t.Fatal(err) + } + if n := hits.Load(); n != 3 { + t.Errorf("%d well-known fetches after expiry, want 3", n) + } +} + +func TestCacheTTL(t *testing.T) { + cases := map[string]time.Duration{ + "": wellKnownDefaultTTL, + "max-age=3600": time.Hour, + "public, max-age=120": 2 * time.Minute, + "max-age=999999999": wellKnownMaxTTL, + "max-age=nonsense": wellKnownDefaultTTL, + } + for header, want := range cases { + if got := cacheTTL(header); got != want { + t.Errorf("cacheTTL(%q) = %v, want %v", header, got, want) + } + } +} diff --git a/server/internal/federation/gate.go b/server/internal/federation/gate.go index a2d8479..1fa6126 100644 --- a/server/internal/federation/gate.go +++ b/server/internal/federation/gate.go @@ -1,12 +1,12 @@ -// Package federation holds the server-to-server surface. In Phase 0 that -// is only the gate: the piece that makes federation a runtime mode rather -// than a build-time one. +// Package federation is the Server-Server protocol, with no knowledge of +// rooms: the gate and the allow/deny policy, X-Matrix request signing, +// server discovery, the signed HTTP client and the key ring. // // Every federation route is registered through Gate.Protect, so flipping // federation.enabled through the admin API takes effect on the next request -// without re-registering routes or restarting the process. The transports -// that the switch also has to start and stop (outbound transaction queue, -// key fetcher, remote media) arrive in Phase 5 as federation.Module. +// without re-registering routes or restarting the process. What the switch +// also has to start and stop — the outbound queue — is the federationapi +// package's Module; see docs/federation.md. package federation import ( diff --git a/server/internal/federation/keys.go b/server/internal/federation/keys.go new file mode 100644 index 0000000..56ae8d0 --- /dev/null +++ b/server/internal/federation/keys.go @@ -0,0 +1,463 @@ +package federation + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "errors" + "fmt" + "log/slog" + "sort" + "sync" + "time" + + "github.com/dariuszjaniak/warp/server/internal/matrix/canonicaljson" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Server keys: publishing ours, and fetching, caching and using everybody +// else's. +// +// Every signature another server makes — on an event, on a request — is +// checked against a key that server publishes at /_matrix/key/v2/server. +// The key ring fetches that document, checks that it is signed by the keys +// it lists (a key response vouches for itself; what makes it trustworthy +// is that it came from the server over TLS, or from a notary we chose to +// trust), and caches each key for as long as the server said it would stay +// valid — but never more than a week, which is the spec's cap on how long +// one fetch may be believed. + +// ErrUnknownKey means a server's key could not be found: not cached, and +// neither the server nor any notary produced it. +var ErrUnknownKey = errors.New("signing key not found") + +// maxKeyValidity caps how far ahead a fetched key's valid_until_ts is +// believed. +const maxKeyValidity = 7 * 24 * time.Hour + +// fetchFailureBackoff stops a server whose key server is down from being +// asked again on every request that needs one of its keys. +const fetchFailureBackoff = time.Minute + +// verifyKey and oldVerifyKey are the key objects of a key response. +type verifyKey struct { + Key string `json:"key"` +} + +type oldVerifyKey struct { + Key string `json:"key"` + ExpiredTS int64 `json:"expired_ts"` +} + +// serverKeyResponse is the document at /_matrix/key/v2/server. +type serverKeyResponse struct { + ServerName string `json:"server_name"` + ValidUntilTS int64 `json:"valid_until_ts"` + VerifyKeys map[string]verifyKey `json:"verify_keys"` + OldVerifyKeys map[string]oldVerifyKey `json:"old_verify_keys"` +} + +// KeyRingOptions configure a KeyRing. +type KeyRingOptions struct { + ServerName string + Key *signing.KeyPair + + // Store caches fetched keys across restarts. + Store storage.FederationStore + + // Client fetches keys. It refuses everything while federation is off, + // which is exactly when no remote key can be needed. + Client *Client + + // Notaries are the servers asked for a key the server itself cannot + // be reached for. + Notaries []string + + // Validity is how long our own key response says our key is valid. + Validity time.Duration + + Logger *slog.Logger + Now func() time.Time +} + +// KeyRing holds this server's key and everybody else's. +type KeyRing struct { + serverName string + key *signing.KeyPair + store storage.FederationStore + client *Client + notaries []string + validity time.Duration + log *slog.Logger + now func() time.Time + + mu sync.Mutex + cache map[string]map[string]storage.ServerKey // server -> key ID -> key + failedAt map[string]time.Time + inflight map[string]*fetchCall +} + +type fetchCall struct { + done chan struct{} + err error +} + +// NewKeyRing builds a key ring. +func NewKeyRing(opts KeyRingOptions) *KeyRing { + if opts.Logger == nil { + opts.Logger = slog.Default() + } + if opts.Now == nil { + opts.Now = time.Now + } + if opts.Validity <= 0 { + opts.Validity = maxKeyValidity + } + return &KeyRing{ + serverName: opts.ServerName, + key: opts.Key, + store: opts.Store, + client: opts.Client, + notaries: opts.Notaries, + validity: opts.Validity, + log: opts.Logger, + now: opts.Now, + cache: map[string]map[string]storage.ServerKey{}, + failedAt: map[string]time.Time{}, + inflight: map[string]*fetchCall{}, + } +} + +// OwnKeyResponse is this server's signed key document, as served at +// /_matrix/key/v2/server and handed out by the notary endpoints. +func (k *KeyRing) OwnKeyResponse() (json.RawMessage, error) { + doc := serverKeyResponse{ + ServerName: k.serverName, + ValidUntilTS: k.now().Add(k.validity).UnixMilli(), + VerifyKeys: map[string]verifyKey{k.key.KeyID(): {Key: k.key.PublicBase64()}}, + OldVerifyKeys: map[string]oldVerifyKey{}, + } + raw, err := json.Marshal(doc) + if err != nil { + return nil, err + } + return signing.SignJSON(k.serverName, k.key, raw) +} + +// PublicKey returns a server's key, fetching it when it is not cached or +// is not known to be valid at atTS. +// +// atTS is the time the key has to have been valid at, in milliseconds: +// the event's origin_server_ts when the room version enforces key +// validity, now for a request. Zero accepts any key the server has ever +// published, which is the rule for events in room versions before 5. +func (k *KeyRing) PublicKey(ctx context.Context, serverName, keyID string, atTS int64) (ed25519.PublicKey, error) { + if serverName == k.serverName { + if keyID == k.key.KeyID() { + return k.key.Public, nil + } + return nil, fmt.Errorf("%w: %s is not a key of this server", ErrUnknownKey, keyID) + } + + if key, ok, err := k.cached(ctx, serverName, keyID); err != nil { + return nil, err + } else if ok && usableAt(key, atTS) { + return decodeKey(key) + } + + if err := k.fetch(ctx, serverName); err != nil { + // A key we already have may still be usable for an event whose + // timestamp the old validity covers; only the refresh failed. + if key, ok, _ := k.cached(ctx, serverName, keyID); ok && usableAt(key, atTS) { + return decodeKey(key) + } + return nil, err + } + key, ok, err := k.cached(ctx, serverName, keyID) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("%w: %s has no key %s", ErrUnknownKey, serverName, keyID) + } + if !usableAt(key, atTS) { + return nil, fmt.Errorf("%w: key %s of %s was not valid at %d", ErrUnknownKey, keyID, serverName, atTS) + } + return decodeKey(key) +} + +// usableAt reports whether a key may verify something signed at atTS. +func usableAt(key storage.ServerKey, atTS int64) bool { + if atTS == 0 { + return true + } + if key.ExpiredTS != 0 { + return atTS <= key.ExpiredTS + } + return atTS <= key.ValidUntilTS +} + +func decodeKey(key storage.ServerKey) (ed25519.PublicKey, error) { + raw, err := signing.DecodeBase64(key.PublicKey) + if err != nil || len(raw) != ed25519.PublicKeySize { + return nil, fmt.Errorf("%w: key %s of %s is malformed", ErrUnknownKey, key.KeyID, key.ServerName) + } + return ed25519.PublicKey(raw), nil +} + +// cached reads a key from memory, then from the store. +func (k *KeyRing) cached(ctx context.Context, serverName, keyID string) (storage.ServerKey, bool, error) { + k.mu.Lock() + key, ok := k.cache[serverName][keyID] + k.mu.Unlock() + if ok { + return key, true, nil + } + + keys, err := k.store.ServerKeys(ctx, serverName, []string{keyID}) + if err != nil { + return storage.ServerKey{}, false, err + } + if len(keys) == 0 { + return storage.ServerKey{}, false, nil + } + k.remember(keys) + return keys[0], true, nil +} + +func (k *KeyRing) remember(keys []storage.ServerKey) { + k.mu.Lock() + defer k.mu.Unlock() + for _, key := range keys { + if k.cache[key.ServerName] == nil { + k.cache[key.ServerName] = map[string]storage.ServerKey{} + } + k.cache[key.ServerName][key.KeyID] = key + } +} + +// fetch refreshes a server's keys, once at a time per server. +func (k *KeyRing) fetch(ctx context.Context, serverName string) error { + k.mu.Lock() + if at, failed := k.failedAt[serverName]; failed && k.now().Sub(at) < fetchFailureBackoff { + k.mu.Unlock() + return fmt.Errorf("%w: fetching the keys of %s failed recently", ErrUnknownKey, serverName) + } + if call, ok := k.inflight[serverName]; ok { + k.mu.Unlock() + select { + case <-call.done: + return call.err + case <-ctx.Done(): + return ctx.Err() + } + } + call := &fetchCall{done: make(chan struct{})} + k.inflight[serverName] = call + k.mu.Unlock() + + call.err = k.fetchKeys(ctx, serverName) + + k.mu.Lock() + delete(k.inflight, serverName) + if call.err != nil { + k.failedAt[serverName] = k.now() + } else { + delete(k.failedAt, serverName) + } + k.mu.Unlock() + close(call.done) + return call.err +} + +// fetchKeys asks the server, then each notary in turn. +func (k *KeyRing) fetchKeys(ctx context.Context, serverName string) error { + raw, err := k.client.ServerKeys(ctx, serverName) + if err == nil { + if err = k.acceptResponse(ctx, serverName, raw, ""); err == nil { + return nil + } + } + k.log.Debug("could not fetch keys directly", "server_name", serverName, "error", err) + + for _, notary := range k.notaries { + if notary == serverName { + continue + } + responses, nerr := k.client.QueryKeys(ctx, notary, map[string]map[string]KeyCriteria{serverName: {}}) + if nerr != nil { + k.log.Debug("notary could not supply keys", "notary", notary, "server_name", serverName, "error", nerr) + continue + } + for _, resp := range responses { + if nerr = k.acceptResponse(ctx, serverName, resp, notary); nerr == nil { + return nil + } + } + } + return fmt.Errorf("%w: no keys for %s: %v", ErrUnknownKey, serverName, err) +} + +// acceptResponse checks a key response and stores its keys. +// +// The response has to be about the server asked for, and every signature +// the server made on it has to come from one of the keys it lists and has +// to verify — a response that cites a key it does not publish is either +// broken or forged. A response relayed by a notary must also carry the +// notary's own signature, which is what the notary is trusted for. +func (k *KeyRing) acceptResponse(ctx context.Context, serverName string, raw []byte, notary string) error { + var doc serverKeyResponse + if err := json.Unmarshal(raw, &doc); err != nil { + return fmt.Errorf("decode key response: %w", err) + } + if doc.ServerName != serverName { + return fmt.Errorf("key response is for %q, not %q", doc.ServerName, serverName) + } + sigs, err := signing.ExtractSignatures(raw) + if err != nil { + return err + } + if len(sigs[serverName]) == 0 { + return fmt.Errorf("key response from %s is not signed by it", serverName) + } + for keyID := range sigs[serverName] { + vk, ok := doc.VerifyKeys[keyID] + if !ok { + return fmt.Errorf("key response from %s is signed with %s, which it does not list", serverName, keyID) + } + pub, err := signing.DecodeBase64(vk.Key) + if err != nil || len(pub) != ed25519.PublicKeySize { + return fmt.Errorf("key %s of %s is malformed", keyID, serverName) + } + if err := signing.VerifyJSON(serverName, keyID, pub, raw); err != nil { + return fmt.Errorf("key response from %s: %w", serverName, err) + } + } + if notary != "" { + if err := k.verifyJSONAt(ctx, notary, 0, raw); err != nil { + return fmt.Errorf("key response for %s relayed by %s: %w", serverName, notary, err) + } + } + + now := k.now() + validUntil := min(doc.ValidUntilTS, now.Add(maxKeyValidity).UnixMilli()) + var keys []storage.ServerKey + for keyID, vk := range doc.VerifyKeys { + keys = append(keys, storage.ServerKey{ + ServerName: serverName, KeyID: keyID, PublicKey: vk.Key, + ValidUntilTS: validUntil, FetchedAtMS: now.UnixMilli(), + }) + } + for keyID, ok := range doc.OldVerifyKeys { + keys = append(keys, storage.ServerKey{ + ServerName: serverName, KeyID: keyID, PublicKey: ok.Key, + ValidUntilTS: ok.ExpiredTS, ExpiredTS: ok.ExpiredTS, FetchedAtMS: now.UnixMilli(), + }) + } + sort.Slice(keys, func(i, j int) bool { return keys[i].KeyID < keys[j].KeyID }) + if err := k.store.StoreServerKeys(ctx, keys); err != nil { + return err + } + k.remember(keys) + return nil +} + +// VerifyJSON checks that a signed object carries a valid signature from +// serverName, made with a key valid now. +func (k *KeyRing) VerifyJSON(ctx context.Context, serverName string, msg []byte) error { + return k.verifyJSONAt(ctx, serverName, k.now().UnixMilli(), msg) +} + +// verifyJSONAt is VerifyJSON with the time the key must be valid at. +// +// One good signature is enough. A server rotating its key signs with the +// new one, and an object it signed with both is still its object if we +// can only fetch one of them. +func (k *KeyRing) verifyJSONAt(ctx context.Context, serverName string, atTS int64, msg []byte) error { + sigs, err := signing.ExtractSignatures(msg) + if err != nil { + return err + } + keyIDs := make([]string, 0, len(sigs[serverName])) + for keyID := range sigs[serverName] { + if algorithm, _, ok := signing.SplitKeyID(keyID); ok && algorithm == signing.KeyIDPrefix { + keyIDs = append(keyIDs, keyID) + } + } + if len(keyIDs) == 0 { + return fmt.Errorf("%w: from %s", signing.ErrNoSignature, serverName) + } + sort.Strings(keyIDs) + + var lastErr error + for _, keyID := range keyIDs { + pub, err := k.PublicKey(ctx, serverName, keyID, atTS) + if err != nil { + lastErr = err + continue + } + if err := signing.VerifyJSON(serverName, keyID, pub, msg); err != nil { + lastErr = err + continue + } + return nil + } + return lastErr +} + +// VerifyEventSignature implements auth.Verifier, and is what every event +// from another server is checked with. +func (k *KeyRing) VerifyEventSignature(ctx context.Context, serverName string, e *event.Event) error { + keyIDs := make([]string, 0, len(e.Signatures()[serverName])) + for keyID := range e.Signatures()[serverName] { + if algorithm, _, ok := signing.SplitKeyID(keyID); ok && algorithm == signing.KeyIDPrefix { + keyIDs = append(keyIDs, keyID) + } + } + if len(keyIDs) == 0 { + return fmt.Errorf("%w: event %s is not signed by %s", signing.ErrNoSignature, e.EventID(), serverName) + } + sort.Strings(keyIDs) + + var atTS int64 + if e.Rules().EnforceSigningKeyValidity { + atTS = e.OriginServerTS() + } + var lastErr error + for _, keyID := range keyIDs { + pub, err := k.PublicKey(ctx, serverName, keyID, atTS) + if err != nil { + lastErr = err + continue + } + if err := e.VerifySignature(serverName, keyID, pub); err != nil { + lastErr = err + continue + } + return nil + } + return lastErr +} + +// NotaryResponse answers a /key/v2/query for this server's own keys. The +// key ring does not relay other servers' responses — it keeps their keys, +// not the documents they came in — so a query about anybody else is +// answered with nothing, which a requester treats as "ask somebody else". +func (k *KeyRing) NotaryResponse(serverName string) ([]json.RawMessage, error) { + if serverName != k.serverName { + return nil, nil + } + own, err := k.OwnKeyResponse() + if err != nil { + return nil, err + } + // The response is already signed by this server; as the notary it is + // the same signature, and canonical JSON keeps it byte-identical. + canonical, err := canonicaljson.Encode(own) + if err != nil { + return nil, err + } + return []json.RawMessage{canonical}, nil +} diff --git a/server/internal/federation/keys_test.go b/server/internal/federation/keys_test.go new file mode 100644 index 0000000..e59a23d --- /dev/null +++ b/server/internal/federation/keys_test.go @@ -0,0 +1,259 @@ +package federation + +import ( + "context" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" + "github.com/dariuszjaniak/warp/server/internal/storage" + _ "github.com/dariuszjaniak/warp/server/internal/storage/drivers" +) + +// testStore opens a migrated SQLite database for the key cache. +func testStore(t *testing.T) storage.FederationStore { + t.Helper() + db, err := storage.Open(t.Context(), storage.Config{ + Driver: storage.DriverSQLite, + ConnectionString: "file:" + strings.ReplaceAll(t.TempDir(), `\`, "/") + "/warp.db", + }, nil) + if err != nil { + t.Fatalf("open: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if err := db.Migrate(t.Context()); err != nil { + t.Fatalf("migrate: %v", err) + } + return db.Federation() +} + +// remote is a fake homeserver: a TLS server answering for a server name, +// with its own key. +type remote struct { + name string + key *signing.KeyPair + srv *httptest.Server + mux *http.ServeMux + keyHits atomic.Int32 + + // validFor is the validity its key response claims. + validFor time.Duration +} + +func newRemote(t *testing.T, name string) *remote { + t.Helper() + key, err := signing.Generate("") + if err != nil { + t.Fatal(err) + } + r := &remote{name: name, key: key, mux: http.NewServeMux(), validFor: time.Hour} + r.mux.HandleFunc("GET /_matrix/key/v2/server", func(w http.ResponseWriter, _ *http.Request) { + r.keyHits.Add(1) + doc, _ := json.Marshal(serverKeyResponse{ + ServerName: r.name, + ValidUntilTS: time.Now().Add(r.validFor).UnixMilli(), + VerifyKeys: map[string]verifyKey{r.key.KeyID(): {Key: r.key.PublicBase64()}}, + OldVerifyKeys: map[string]oldVerifyKey{}, + }) + signed, _ := signing.SignJSON(r.name, r.key, doc) + _, _ = w.Write(signed) + }) + r.srv = httptest.NewTLSServer(r.mux) + t.Cleanup(r.srv.Close) + return r +} + +// clientFor builds a client that resolves every remote in the list to its +// test server. The Host is "example.com", the name the test certificate is +// issued for, and the address is the listener: the dial-target path is +// exercised on every request. +func clientFor(t *testing.T, serverName string, key *signing.KeyPair, remotes ...*remote) *Client { + t.Helper() + byName := map[string]*remote{} + for _, r := range remotes { + byName[r.name] = r + } + var tlsConfig = remotes[0].srv.Client().Transport.(*http.Transport).TLSClientConfig.Clone() + return NewClient(ClientOptions{ + ServerName: serverName, + Key: key, + TLSConfig: tlsConfig, + Enabled: func() bool { return true }, + Resolve: func(_ context.Context, name string) (Resolved, error) { + r, ok := byName[name] + if !ok { + return Resolved{}, errors.New("no such test server") + } + _, port, _ := net.SplitHostPort(r.srv.Listener.Addr().String()) + return Resolved{Addresses: []string{r.srv.Listener.Addr().String()}, Host: "example.com:" + port}, nil + }, + }) +} + +func newKeyRing(t *testing.T, name string, remotes ...*remote) (*KeyRing, *signing.KeyPair) { + t.Helper() + key, err := signing.Generate("") + if err != nil { + t.Fatal(err) + } + return NewKeyRing(KeyRingOptions{ + ServerName: name, + Key: key, + Store: testStore(t), + Client: clientFor(t, name, key, remotes...), + }), key +} + +func TestKeyRingFetchesAndCaches(t *testing.T) { + r := newRemote(t, "remote.example") + ring, _ := newKeyRing(t, "warp.example", r) + + pub, err := ring.PublicKey(t.Context(), "remote.example", r.key.KeyID(), time.Now().UnixMilli()) + if err != nil { + t.Fatalf("PublicKey: %v", err) + } + if !pub.Equal(r.key.Public) { + t.Error("fetched the wrong key") + } + for range 3 { + if _, err := ring.PublicKey(t.Context(), "remote.example", r.key.KeyID(), time.Now().UnixMilli()); err != nil { + t.Fatal(err) + } + } + if n := r.keyHits.Load(); n != 1 { + t.Errorf("%d key fetches, want 1", n) + } + + // A key needed for a time past its validity is fetched again, in case + // the server has extended it. + if _, err := ring.PublicKey(t.Context(), "remote.example", r.key.KeyID(), + time.Now().Add(2*time.Hour).UnixMilli()); err == nil { + t.Error("a key was accepted for a time past its validity") + } + if n := r.keyHits.Load(); n != 2 { + t.Errorf("%d key fetches after asking past validity, want 2", n) + } + + // Validity is capped at a week, whatever the server claims. + r.validFor = 365 * 24 * time.Hour + ring2, _ := newKeyRing(t, "warp.example", r) + if _, err := ring2.PublicKey(t.Context(), "remote.example", r.key.KeyID(), + time.Now().Add(30*24*time.Hour).UnixMilli()); err == nil { + t.Error("a key was believed for longer than a week") + } +} + +func TestKeyRingRefusesAForgedResponse(t *testing.T) { + r := newRemote(t, "remote.example") + impostor, _ := signing.Generate("") + r.mux = http.NewServeMux() + r.mux.HandleFunc("GET /_matrix/key/v2/server", func(w http.ResponseWriter, _ *http.Request) { + // Lists the real key but is signed with another one. + doc, _ := json.Marshal(serverKeyResponse{ + ServerName: r.name, + ValidUntilTS: time.Now().Add(time.Hour).UnixMilli(), + VerifyKeys: map[string]verifyKey{r.key.KeyID(): {Key: impostor.PublicBase64()}}, + }) + signed, _ := signing.SignJSON(r.name, r.key, doc) + _, _ = w.Write(signed) + }) + r.srv.Config.Handler = r.mux + + ring, _ := newKeyRing(t, "warp.example", r) + if _, err := ring.PublicKey(t.Context(), "remote.example", r.key.KeyID(), 0); !errors.Is(err, ErrUnknownKey) { + t.Errorf("PublicKey with a forged response = %v, want ErrUnknownKey", err) + } +} + +func TestOwnKeyResponseVerifies(t *testing.T) { + ring, key := newKeyRing(t, "warp.example", newRemote(t, "unused.example")) + doc, err := ring.OwnKeyResponse() + if err != nil { + t.Fatal(err) + } + if err := signing.VerifyJSON("warp.example", key.KeyID(), key.Public, doc); err != nil { + t.Errorf("our own key response does not verify: %v", err) + } + var parsed serverKeyResponse + if err := json.Unmarshal(doc, &parsed); err != nil { + t.Fatal(err) + } + if parsed.VerifyKeys[key.KeyID()].Key != key.PublicBase64() { + t.Errorf("verify_keys = %+v", parsed.VerifyKeys) + } +} + +func TestSignedRequestIsAuthenticatedEndToEnd(t *testing.T) { + // The remote here is the *sender*: a key server for remote.example, + // and a client signing as remote.example to warp.example. + sender := newRemote(t, "remote.example") + receiver := newRemote(t, "warp.example") + ring := NewKeyRing(KeyRingOptions{ + ServerName: "warp.example", + Key: receiver.key, + Store: testStore(t), + Client: clientFor(t, "warp.example", receiver.key, sender), + }) + + var gotOrigin atomic.Value + receiver.mux.HandleFunc("PUT /_matrix/federation/v1/send/{txn}", func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + origin, err := ring.AuthenticateRequest(r.Context(), r, body) + if err != nil { + http.Error(w, `{"errcode":"M_UNAUTHORIZED","error":"`+err.Error()+`"}`, http.StatusUnauthorized) + return + } + gotOrigin.Store(origin) + _, _ = w.Write([]byte(`{"pdus":{}}`)) + }) + + client := clientFor(t, "remote.example", sender.key, receiver) + if _, err := client.SendTransaction(t.Context(), "warp.example", "txn/1 with spaces", Transaction{ + Origin: "remote.example", OriginServerTS: 1, PDUs: []json.RawMessage{}, + }); err != nil { + t.Fatalf("SendTransaction: %v", err) + } + if got, _ := gotOrigin.Load().(string); got != "remote.example" { + t.Errorf("authenticated origin = %q", got) + } + + // A request signed by a key the origin does not publish is refused. + stranger, _ := signing.Generate("") + forged := clientFor(t, "remote.example", stranger, receiver) + _, err := forged.SendTransaction(t.Context(), "warp.example", "2", Transaction{Origin: "remote.example"}) + var httpErr *HTTPError + if !errors.As(err, &httpErr) || httpErr.Status != http.StatusUnauthorized { + t.Errorf("forged request = %v, want 401", err) + } +} + +func TestClientRefusesWhileDisabledOrDenied(t *testing.T) { + r := newRemote(t, "remote.example") + key, _ := signing.Generate("") + enabled := false + client := clientFor(t, "warp.example", key, r) + client.enabled = func() bool { return enabled } + + if _, err := client.Version(t.Context(), "remote.example"); !errors.Is(err, ErrDisabled) { + t.Errorf("Version while disabled = %v, want ErrDisabled", err) + } + enabled = true + client.policy = NewPolicy(nil, []string{"remote.example"}) + if _, err := client.Version(t.Context(), "remote.example"); !errors.Is(err, ErrNotAllowed) { + t.Errorf("Version to a denied server = %v, want ErrNotAllowed", err) + } + if _, err := client.Version(t.Context(), "warp.example"); !errors.Is(err, ErrSelf) { + t.Errorf("Version to ourselves = %v, want ErrSelf", err) + } + if r.keyHits.Load() != 0 { + t.Error("a refused request reached the network") + } +} diff --git a/server/internal/federation/requestauth.go b/server/internal/federation/requestauth.go new file mode 100644 index 0000000..d18e56b --- /dev/null +++ b/server/internal/federation/requestauth.go @@ -0,0 +1,68 @@ +package federation + +import ( + "context" + "fmt" + "net/http" +) + +// AuthenticateRequest checks the X-Matrix signature on an inbound request +// and returns the server it came from. +// +// content is the request body, already read; the signature covers it, so +// the caller reads it once and hands the same bytes here and to its own +// decoder. A request may carry one Authorization header per key it is +// signed with, and any one that verifies is enough. +func (k *KeyRing) AuthenticateRequest(ctx context.Context, r *http.Request, content []byte) (string, error) { + headers := r.Header.Values("Authorization") + if len(headers) == 0 { + return "", fmt.Errorf("%w: no Authorization header", ErrBadAuthorization) + } + + var ( + origin string + lastErr error + ) + for _, h := range headers { + x, err := ParseXMatrix(h) + if err != nil { + lastErr = err + continue + } + if origin != "" && x.Origin != origin { + // Two headers naming two origins is one request claiming to + // be from two servers; neither claim is worth checking. + return "", fmt.Errorf("%w: headers name both %s and %s", ErrBadAuthorization, origin, x.Origin) + } + origin = x.Origin + + if x.Destination != "" && x.Destination != k.serverName { + lastErr = fmt.Errorf("%w: addressed to %s, not to this server", ErrBadAuthorization, x.Destination) + continue + } + if x.Origin == k.serverName { + lastErr = fmt.Errorf("%w: a request claiming to be from this server", ErrBadAuthorization) + continue + } + + // The destination in the signed object is whatever the sender put + // in the header, or this server's name when it put nothing: a + // sender old enough to leave it out signed with it absent from + // the header, not from the object. + destination := x.Destination + if destination == "" { + destination = k.serverName + } + signed, err := requestSignedJSON(x, destination, r.Method, r.RequestURI, content) + if err != nil { + lastErr = err + continue + } + if err := k.VerifyJSON(ctx, x.Origin, signed); err != nil { + lastErr = fmt.Errorf("%w: %v", ErrBadAuthorization, err) + continue + } + return x.Origin, nil + } + return "", lastErr +} diff --git a/server/internal/federation/types.go b/server/internal/federation/types.go new file mode 100644 index 0000000..9b1f011 --- /dev/null +++ b/server/internal/federation/types.go @@ -0,0 +1,151 @@ +package federation + +import "encoding/json" + +// The Server-Server API's request and response bodies, shared by the +// client that sends them and the handlers that answer them. +// +// PDUs travel as json.RawMessage throughout. Decoding one needs its room +// version, and only the roomserver knows that; anything that parsed a PDU +// on the way through here would have to guess. + +// Transaction is the body of PUT /send/{txnId}, and the shape /event and +// /backfill answer with. +type Transaction struct { + Origin string `json:"origin"` + OriginServerTS int64 `json:"origin_server_ts"` + PDUs []json.RawMessage `json:"pdus"` + EDUs []EDU `json:"edus,omitempty"` +} + +// EDU is an ephemeral data unit: typing, receipts, presence, to-device. +type EDU struct { + Type string `json:"edu_type"` + Content json.RawMessage `json:"content"` +} + +// The EDU types Warp sends and receives. +const ( + EDUTyping = "m.typing" + EDUReceipt = "m.receipt" + EDUDirectToDev = "m.direct_to_device" + EDUPresence = "m.presence" + EDUDeviceListUp = "m.device_list_update" +) + +// SendTransactionResponse is the answer to PUT /send: a result per PDU. +type SendTransactionResponse struct { + PDUs map[string]PDUResult `json:"pdus"` +} + +// PDUResult is the outcome for one PDU. An empty Error means it was +// accepted. +type PDUResult struct { + Error string `json:"error,omitempty"` +} + +// MakeJoinResponse is the answer to GET /make_join (and /make_leave): an +// unsigned event template the joining server fills in and signs. +type MakeJoinResponse struct { + RoomVersion string `json:"room_version,omitempty"` + Event json.RawMessage `json:"event"` +} + +// SendJoinResponse is the v2 answer to PUT /send_join: the room as of the +// join. +type SendJoinResponse struct { + Origin string `json:"origin"` + State []json.RawMessage `json:"state"` + AuthChain []json.RawMessage `json:"auth_chain"` + + // Event is the join as the resident server signed it, for a join + // under the restricted join rule that it had to countersign. + Event json.RawMessage `json:"event,omitempty"` + + MembersOmitted bool `json:"members_omitted,omitempty"` + ServersInRoom []string `json:"servers_in_room,omitempty"` +} + +// InviteRequest is the v2 body of PUT /invite. +type InviteRequest struct { + RoomVersion string `json:"room_version"` + Event json.RawMessage `json:"event"` + InviteRoomState []json.RawMessage `json:"invite_room_state,omitempty"` +} + +// InviteResponse is the answer to PUT /invite: the invite, countersigned. +type InviteResponse struct { + Event json.RawMessage `json:"event"` +} + +// StateIDsResponse is the answer to GET /state_ids. +type StateIDsResponse struct { + PDUIDs []string `json:"pdu_ids"` + AuthChainIDs []string `json:"auth_chain_ids"` +} + +// StateResponse is the answer to GET /state. +type StateResponse struct { + PDUs []json.RawMessage `json:"pdus"` + AuthChain []json.RawMessage `json:"auth_chain"` +} + +// EventAuthResponse is the answer to GET /event_auth. +type EventAuthResponse struct { + AuthChain []json.RawMessage `json:"auth_chain"` +} + +// MissingEventsRequest is the body of POST /get_missing_events. +type MissingEventsRequest struct { + EarliestEvents []string `json:"earliest_events"` + LatestEvents []string `json:"latest_events"` + Limit int `json:"limit,omitempty"` + MinDepth int64 `json:"min_depth,omitempty"` +} + +// MissingEventsResponse is the answer to POST /get_missing_events. +type MissingEventsResponse struct { + Events []json.RawMessage `json:"events"` +} + +// DirectoryResponse is the answer to GET /query/directory. +type DirectoryResponse struct { + RoomID string `json:"room_id"` + Servers []string `json:"servers"` +} + +// VersionResponse is the answer to GET /version. +type VersionResponse struct { + Server struct { + Name string `json:"name"` + Version string `json:"version"` + } `json:"server"` +} + +// TypingContent is the content of an m.typing EDU. +type TypingContent struct { + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + Typing bool `json:"typing"` +} + +// ReceiptContent is the content of an m.receipt EDU: room ID, then receipt +// type, then user ID. +type ReceiptContent map[string]map[string]map[string]ReceiptData + +// ReceiptData is one user's receipt in an m.receipt EDU. +type ReceiptData struct { + EventIDs []string `json:"event_ids"` + Data struct { + TS int64 `json:"ts"` + ThreadID string `json:"thread_id,omitempty"` + } `json:"data"` +} + +// DirectToDeviceContent is the content of an m.direct_to_device EDU. +type DirectToDeviceContent struct { + Sender string `json:"sender"` + Type string `json:"type"` + MessageID string `json:"message_id"` + Messages map[string]map[string]json.RawMessage `json:"messages"` +} diff --git a/server/internal/federation/xmatrix.go b/server/internal/federation/xmatrix.go new file mode 100644 index 0000000..2fe87d7 --- /dev/null +++ b/server/internal/federation/xmatrix.go @@ -0,0 +1,239 @@ +package federation + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/dariuszjaniak/warp/server/internal/matrix/canonicaljson" + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" +) + +// The X-Matrix authorization scheme: how one server proves to another that +// a request came from it. +// +// The sender signs a small JSON object naming the request — method, URI, +// origin, destination and, when there is one, the JSON body — and sends +// the signature in the Authorization header: +// +// Authorization: X-Matrix origin="a.example",destination="b.example",key="ed25519:k",sig="..." +// +// The receiver rebuilds the same object from the request it actually got +// and checks the signature against the origin's published key. Nothing in +// the header is trusted until that check passes: the origin it names is a +// claim, and it is exactly the claim being checked. + +// ErrBadAuthorization is the root of every X-Matrix parsing and +// verification failure. Each one is answered with M_UNAUTHORIZED. +var ErrBadAuthorization = errors.New("invalid X-Matrix authorization") + +// XMatrix is one parsed X-Matrix header. +type XMatrix struct { + Origin string + + // Destination is the server the request was addressed to. Older + // servers leave it out, and a receiver then has to assume it was + // itself. + Destination string + + // Key is the full key ID, algorithm included. + Key string + + // Sig is the unpadded base64 signature. + Sig string +} + +// maxAuthorizationLength bounds the header before anything is parsed. A +// real one is a couple of hundred bytes; this leaves room for long server +// names without letting a client make the parser chew on a megabyte. +const maxAuthorizationLength = 4096 + +// ParseXMatrix parses an Authorization header value. +// +// The grammar is the spec's: the scheme, then comma-separated key=value +// parameters whose values may be quoted. Quoted values may contain +// backslash escapes, and unknown parameters are ignored, so that a header +// carrying something a newer server added still parses. +func ParseXMatrix(header string) (XMatrix, error) { + if len(header) > maxAuthorizationLength { + return XMatrix{}, fmt.Errorf("%w: header is %d bytes", ErrBadAuthorization, len(header)) + } + scheme, rest, ok := strings.Cut(strings.TrimSpace(header), " ") + if !ok || !strings.EqualFold(scheme, "X-Matrix") { + return XMatrix{}, fmt.Errorf("%w: not an X-Matrix header", ErrBadAuthorization) + } + + var x XMatrix + params, err := parseAuthParams(rest) + if err != nil { + return XMatrix{}, err + } + for name, value := range params { + switch strings.ToLower(name) { + case "origin": + x.Origin = value + case "destination": + x.Destination = value + case "key": + x.Key = value + case "sig": + x.Sig = value + } + } + + switch { + case x.Origin == "": + return XMatrix{}, fmt.Errorf("%w: no origin", ErrBadAuthorization) + case x.Key == "": + return XMatrix{}, fmt.Errorf("%w: no key", ErrBadAuthorization) + case x.Sig == "": + return XMatrix{}, fmt.Errorf("%w: no sig", ErrBadAuthorization) + } + if _, _, ok := signing.SplitKeyID(x.Key); !ok { + return XMatrix{}, fmt.Errorf("%w: key %q is not algorithm:id", ErrBadAuthorization, x.Key) + } + return x, nil +} + +// parseAuthParams splits `a="x",b=y` into its parameters. +func parseAuthParams(s string) (map[string]string, error) { + params := map[string]string{} + i := 0 + for { + // Skip separators and whitespace between parameters. + for i < len(s) && (s[i] == ',' || s[i] == ' ' || s[i] == '\t') { + i++ + } + if i >= len(s) { + return params, nil + } + + eq := strings.IndexByte(s[i:], '=') + if eq <= 0 { + return nil, fmt.Errorf("%w: parameter without a value", ErrBadAuthorization) + } + name := strings.TrimSpace(s[i : i+eq]) + if name == "" || strings.ContainsAny(name, " \t\",") { + return nil, fmt.Errorf("%w: malformed parameter name %q", ErrBadAuthorization, name) + } + i += eq + 1 + + var value string + if i < len(s) && s[i] == '"' { + var b strings.Builder + i++ + closed := false + for i < len(s) { + c := s[i] + if c == '\\' && i+1 < len(s) { + b.WriteByte(s[i+1]) + i += 2 + continue + } + if c == '"' { + closed = true + i++ + break + } + b.WriteByte(c) + i++ + } + if !closed { + return nil, fmt.Errorf("%w: unterminated quoted value", ErrBadAuthorization) + } + value = b.String() + } else { + end := strings.IndexByte(s[i:], ',') + if end < 0 { + end = len(s) - i + } + value = strings.TrimSpace(s[i : i+end]) + i += end + } + + if _, dup := params[name]; dup { + // Two origins in one header is not something to resolve by + // picking one: whichever we picked, the other server meant + // something else. + return nil, fmt.Errorf("%w: parameter %q given twice", ErrBadAuthorization, name) + } + params[name] = value + } +} + +// String renders the header value. Every parameter is quoted, which the +// spec allows and which every implementation accepts. +func (x XMatrix) String() string { + quote := func(s string) string { + return `"` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(s) + `"` + } + return "X-Matrix origin=" + quote(x.Origin) + + ",destination=" + quote(x.Destination) + + ",key=" + quote(x.Key) + + ",sig=" + quote(x.Sig) +} + +// requestJSON is the object an X-Matrix signature covers. +type requestJSON struct { + Method string `json:"method"` + URI string `json:"uri"` + Origin string `json:"origin"` + Destination string `json:"destination"` + Content json.RawMessage `json:"content,omitempty"` +} + +// signableRequest builds the bytes a request signature covers. +// +// uri is the request target exactly as it was sent — path and query, still +// escaped. Both ends must agree on it byte for byte, which is why the +// receiver reads it from the raw request line rather than reassembling it +// from a parsed URL. +func signableRequest(method, uri, origin, destination string, content []byte) ([]byte, error) { + obj := requestJSON{Method: method, URI: uri, Origin: origin, Destination: destination} + if len(content) > 0 { + obj.Content = content + } + raw, err := json.Marshal(obj) + if err != nil { + return nil, fmt.Errorf("encode request for signing: %w", err) + } + return canonicaljson.Encode(raw) +} + +// SignRequest produces the X-Matrix header for an outbound request. +func SignRequest(key *signing.KeyPair, origin, destination, method, uri string, content []byte) (XMatrix, error) { + msg, err := signableRequest(method, uri, origin, destination, content) + if err != nil { + return XMatrix{}, err + } + signed, err := signing.SignJSON(origin, key, msg) + if err != nil { + return XMatrix{}, err + } + sigs, err := signing.ExtractSignatures(signed) + if err != nil { + return XMatrix{}, err + } + sig, _ := sigs.Get(origin, key.KeyID()) + return XMatrix{Origin: origin, Destination: destination, Key: key.KeyID(), Sig: sig}, nil +} + +// requestSignedJSON rebuilds the signed object an inbound header claims to +// cover, with the header's signature attached, ready for VerifyJSON. +func requestSignedJSON(x XMatrix, destination, method, uri string, content []byte) ([]byte, error) { + msg, err := signableRequest(method, uri, x.Origin, destination, content) + if err != nil { + return nil, err + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(msg, &obj); err != nil { + return nil, err + } + sigs, err := json.Marshal(signing.Signatures{x.Origin: {x.Key: x.Sig}}) + if err != nil { + return nil, err + } + obj["signatures"] = sigs + return json.Marshal(obj) +} diff --git a/server/internal/federation/xmatrix_test.go b/server/internal/federation/xmatrix_test.go new file mode 100644 index 0000000..5d9b817 --- /dev/null +++ b/server/internal/federation/xmatrix_test.go @@ -0,0 +1,122 @@ +package federation + +import ( + "errors" + "strings" + "testing" + + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" +) + +func TestParseXMatrix(t *testing.T) { + cases := []struct { + name string + header string + want XMatrix + bad bool + }{ + { + name: "quoted", + header: `X-Matrix origin="a.example",destination="b.example",key="ed25519:k1",sig="c2ln"`, + want: XMatrix{Origin: "a.example", Destination: "b.example", Key: "ed25519:k1", Sig: "c2ln"}, + }, + { + name: "unquoted and spaced", + header: `X-Matrix origin=a.example, key=ed25519:k1 ,sig=c2ln`, + want: XMatrix{Origin: "a.example", Key: "ed25519:k1", Sig: "c2ln"}, + }, + { + name: "escapes and an unknown parameter", + header: `x-matrix origin="a\"b",future="x,y",key="ed25519:k",sig="s"`, + want: XMatrix{Origin: `a"b`, Key: "ed25519:k", Sig: "s"}, + }, + {name: "wrong scheme", header: `Bearer abc`, bad: true}, + {name: "no origin", header: `X-Matrix key="ed25519:k",sig="s"`, bad: true}, + {name: "no sig", header: `X-Matrix origin="a",key="ed25519:k"`, bad: true}, + {name: "key without an algorithm", header: `X-Matrix origin="a",key="k",sig="s"`, bad: true}, + {name: "unterminated quote", header: `X-Matrix origin="a,key="ed25519:k",sig="s"`, bad: true}, + {name: "duplicate origin", header: `X-Matrix origin="a",origin="b",key="ed25519:k",sig="s"`, bad: true}, + {name: "too long", header: "X-Matrix origin=" + strings.Repeat("a", 5000) + `,key="ed25519:k",sig="s"`, bad: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := ParseXMatrix(tc.header) + if tc.bad { + if !errors.Is(err, ErrBadAuthorization) { + t.Fatalf("ParseXMatrix(%q) = %+v, %v; want ErrBadAuthorization", tc.header, got, err) + } + return + } + if err != nil { + t.Fatalf("ParseXMatrix(%q): %v", tc.header, err) + } + if got != tc.want { + t.Errorf("ParseXMatrix(%q) = %+v, want %+v", tc.header, got, tc.want) + } + }) + } +} + +func TestXMatrixStringRoundTrips(t *testing.T) { + x := XMatrix{Origin: `we"ird`, Destination: `b\c`, Key: "ed25519:k", Sig: "c2ln"} + got, err := ParseXMatrix(x.String()) + if err != nil { + t.Fatalf("ParseXMatrix(%q): %v", x.String(), err) + } + if got != x { + t.Errorf("round trip = %+v, want %+v", got, x) + } +} + +func TestSignedRequestVerifies(t *testing.T) { + key, err := signing.Generate("k1") + if err != nil { + t.Fatal(err) + } + content := []byte(`{"b":2,"a":1}`) + x, err := SignRequest(key, "a.example", "b.example", "PUT", "/_matrix/federation/v1/send/1?x=%21", content) + if err != nil { + t.Fatalf("SignRequest: %v", err) + } + + verify := func(method, uri string, body []byte) error { + signed, err := requestSignedJSON(x, "b.example", method, uri, body) + if err != nil { + return err + } + return signing.VerifyJSON("a.example", x.Key, key.Public, signed) + } + if err := verify("PUT", "/_matrix/federation/v1/send/1?x=%21", []byte(`{"a":1,"b":2}`)); err != nil { + t.Errorf("the request as signed does not verify: %v", err) + } + // Each part of the request is covered. + if err := verify("POST", "/_matrix/federation/v1/send/1?x=%21", content); err == nil { + t.Error("a different method verified") + } + if err := verify("PUT", "/_matrix/federation/v1/send/1?x=!", content); err == nil { + t.Error("a differently escaped URI verified") + } + if err := verify("PUT", "/_matrix/federation/v1/send/1?x=%21", []byte(`{"a":1}`)); err == nil { + t.Error("a different body verified") + } +} + +func FuzzParseXMatrix(f *testing.F) { + f.Add(`X-Matrix origin="a.example",destination="b.example",key="ed25519:k1",sig="c2ln"`) + f.Add(`X-Matrix origin=a,key=ed25519:k,sig=s`) + f.Add(`X-Matrix origin="\\",key="ed25519:k",sig="\"`) + f.Fuzz(func(t *testing.T, header string) { + x, err := ParseXMatrix(header) + if err != nil { + return + } + // Whatever parses must survive being written back out. + again, err := ParseXMatrix(x.String()) + if err != nil { + t.Fatalf("re-parsing %q (from %q): %v", x.String(), header, err) + } + if again != x { + t.Fatalf("round trip of %q changed %+v into %+v", header, x, again) + } + }) +} diff --git a/server/internal/federationapi/edus.go b/server/internal/federationapi/edus.go new file mode 100644 index 0000000..ed2c5d8 --- /dev/null +++ b/server/internal/federationapi/edus.go @@ -0,0 +1,98 @@ +package federationapi + +import ( + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "time" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" +) + +// EDUs turns the sync API's local typing, receipts and to-device messages +// into queued EDUs. It implements syncapi.Outbound. +type EDUs struct { + queue *Queue + rooms RoomServers +} + +// NewEDUs builds the adapter. +func NewEDUs(queue *Queue, rooms RoomServers) *EDUs { + return &EDUs{queue: queue, rooms: rooms} +} + +// roomServers is everybody in a room but this server; the queue drops +// this server anyway, and an error is logged rather than returned, because +// a typing notification that does not go out is not worth failing the +// client's request over. +func (e *EDUs) roomServers(ctx context.Context, roomID string) []string { + servers, err := e.rooms.JoinedServers(ctx, roomID) + if err != nil { + e.queue.log.Warn("could not read a room's servers for an EDU", "room_id", roomID, "error", err) + return nil + } + return servers +} + +// Typing implements syncapi.Outbound. +func (e *EDUs) Typing(ctx context.Context, roomID, userID string, typing bool, _ time.Duration) { + if !e.queue.Running() { + return + } + content, _ := json.Marshal(federation.TypingContent{RoomID: roomID, UserID: userID, Typing: typing}) + e.queue.QueueEDU(ctx, e.roomServers(ctx, roomID), federation.EDU{Type: federation.EDUTyping, Content: content}) +} + +// Receipt implements syncapi.Outbound. +func (e *EDUs) Receipt(ctx context.Context, roomID, userID, receiptType, eventID, threadID string, ts int64) { + if !e.queue.Running() || receiptType != "m.read" { + return + } + var data federation.ReceiptData + data.EventIDs = []string{eventID} + data.Data.TS = ts + data.Data.ThreadID = threadID + content, _ := json.Marshal(federation.ReceiptContent{roomID: {receiptType: {userID: data}}}) + e.queue.QueueEDU(ctx, e.roomServers(ctx, roomID), federation.EDU{Type: federation.EDUReceipt, Content: content}) +} + +// ToDevice implements syncapi.Outbound: one m.direct_to_device per +// destination, carrying that server's recipients only. +func (e *EDUs) ToDevice(ctx context.Context, sender, eventType string, messages map[string]map[string]json.RawMessage) { + if !e.queue.Running() { + return + } + byServer := map[string]map[string]map[string]json.RawMessage{} + for userID, devices := range messages { + server, err := id.ServerNameOf(userID) + if err != nil { + continue + } + if byServer[server] == nil { + byServer[server] = map[string]map[string]json.RawMessage{} + } + byServer[server][userID] = devices + } + for server, recipients := range byServer { + content, err := json.Marshal(federation.DirectToDeviceContent{ + Sender: sender, + Type: eventType, + MessageID: messageID(), + Messages: recipients, + }) + if err != nil { + continue + } + e.queue.QueueEDU(ctx, []string{server}, federation.EDU{Type: federation.EDUDirectToDev, Content: content}) + } +} + +// messageID is the per-message identifier a receiving server deduplicates +// a to-device message by. +func messageID() string { + buf := make([]byte, 12) + _, _ = rand.Read(buf) + return base64.RawURLEncoding.EncodeToString(buf) +} diff --git a/server/internal/federationapi/module.go b/server/internal/federationapi/module.go new file mode 100644 index 0000000..6e4c924 --- /dev/null +++ b/server/internal/federationapi/module.go @@ -0,0 +1,96 @@ +// Package federationapi is the Server-Server API as a running component: +// the inbound endpoints other servers call, the outbound queue that sends +// them this server's events, and the Module that starts and stops it all +// with the federation switch. +// +// The protocol itself — X-Matrix signing, server discovery, the HTTP +// client, keys — is in package federation. This package is where that +// protocol meets the roomserver and the sync API, which is why it is +// separate: the roomserver uses the protocol, and this package uses the +// roomserver. +package federationapi + +import ( + "context" + "log/slog" + "sync" +) + +// Module is the switchable part of federation: everything that runs in the +// background and talks to other servers on its own initiative. +// +// The inbound endpoints are not in it. They are behind the gate, which +// reads the switch on every request, so they need nothing started or +// stopped. What does is the outbound queue — the only thing in Warp that +// opens connections to other servers unprompted — and turning federation +// off stops it mid-flight, leaving what it had not sent in the database for +// when federation comes back on. +// +// Requests a component makes on a client's behalf — a join, a key fetch, +// a profile lookup — need nothing stopping either: the client refuses +// every request while the switch is off. +type Module struct { + queue *Queue + log *slog.Logger + + mu sync.Mutex + running bool +} + +// NewModule builds a stopped module. +func NewModule(queue *Queue, log *slog.Logger) *Module { + if log == nil { + log = slog.Default() + } + return &Module{queue: queue, log: log} +} + +// Start brings federation up. Starting a running module does nothing. +func (m *Module) Start(ctx context.Context) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.running { + return nil + } + if err := m.queue.Start(ctx); err != nil { + m.queue.Stop() + return err + } + m.running = true + m.log.Info("federation started") + return nil +} + +// Stop takes federation down and waits for in-flight sends to finish or +// be cancelled. Stopping a stopped module does nothing. +func (m *Module) Stop() { + m.mu.Lock() + defer m.mu.Unlock() + if !m.running { + return + } + m.queue.Stop() + m.running = false + m.log.Info("federation stopped") +} + +// Apply starts or stops the module to match the switch. +func (m *Module) Apply(ctx context.Context, enabled bool) { + if enabled { + if err := m.Start(ctx); err != nil { + m.log.Error("could not start federation", "error", err) + } + return + } + m.Stop() +} + +// Running reports whether the module is up. +func (m *Module) Running() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.running +} + +// Queue exposes the outbound queue, for the admin API. +func (m *Module) Queue() *Queue { return m.queue } diff --git a/server/internal/federationapi/queue.go b/server/internal/federationapi/queue.go new file mode 100644 index 0000000..0c3c79d --- /dev/null +++ b/server/internal/federationapi/queue.go @@ -0,0 +1,563 @@ +package federationapi + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "math/rand/v2" + "slices" + "strconv" + "sync" + "time" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// The outbound queue: every event and EDU this server owes another one. +// +// It has three moving parts. The roomserver hands it every persisted event +// through EventPersisted, which only appends to an in-memory list — it is +// called with the room's write lock held and must not touch the database. +// A dispatcher drains that list, works out each event's destinations and +// writes one queue row per destination. And one worker per destination +// with something queued sends it, in transactions of up to 50 PDUs and +// 100 EDUs, backing off exponentially when the destination fails. +// +// The rows are deleted only once the destination has answered, so a crash +// or a restart loses nothing that had reached the database, and a +// destination that is down gets everything it missed when it comes back. +// What it does lose is the dispatcher's in-memory list: the events +// persisted in the instant before a crash. A remote server notices those +// as gaps the next time it hears from this one, and fetches them. + +// Transaction limits, from the spec. +const ( + maxPDUsPerTxn = 50 + maxEDUsPerTxn = 100 +) + +// Backoff bounds. The first retry comes quickly, because most failures are +// a restart on the other side; the cap is long, because a server that has +// been gone for a day is not coming back in the next minute, and every +// retry costs a connection attempt. +const ( + defaultMinBackoff = 10 * time.Second + defaultMaxBackoff = 6 * time.Hour +) + +// sendTimeout bounds one transaction. +const sendTimeout = 2 * time.Minute + +// Sender is the part of the federation client the queue uses. +type Sender interface { + SendTransaction(ctx context.Context, destination, txnID string, txn federation.Transaction) (federation.SendTransactionResponse, error) +} + +// RoomServers says which servers are in a room. +type RoomServers interface { + JoinedServers(ctx context.Context, roomID string) ([]string, error) +} + +// QueueOptions configure a Queue. +type QueueOptions struct { + ServerName string + Store storage.FederationStore + Sender Sender + Rooms RoomServers + Policy *federation.Policy + + MinBackoff time.Duration + MaxBackoff time.Duration + + Logger *slog.Logger + Now func() time.Time +} + +// Queue is the outbound queue. +type Queue struct { + serverName string + store storage.FederationStore + sender Sender + rooms RoomServers + policy *federation.Policy + minBackoff time.Duration + maxBackoff time.Duration + log *slog.Logger + now func() time.Time + + // txnPrefix makes transaction IDs unique across restarts: a remote + // server deduplicates by (origin, txn ID), and an ID reused with + // different contents would be answered from its cache and the events + // in it never seen. + txnPrefix string + + mu sync.Mutex + running bool + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + incoming []roomserver.Notification + wake chan struct{} + workers map[string]*worker + txnSeq uint64 +} + +// worker sends one destination's queue. +type worker struct { + destination string + wake chan struct{} +} + +// NewQueue builds a stopped queue. +func NewQueue(opts QueueOptions) *Queue { + if opts.Logger == nil { + opts.Logger = slog.Default() + } + if opts.Now == nil { + opts.Now = time.Now + } + if opts.MinBackoff <= 0 { + opts.MinBackoff = defaultMinBackoff + } + if opts.MaxBackoff < opts.MinBackoff { + opts.MaxBackoff = max(defaultMaxBackoff, opts.MinBackoff) + } + if opts.Policy == nil { + opts.Policy = federation.NewPolicy(nil, nil) + } + return &Queue{ + serverName: opts.ServerName, + store: opts.Store, + sender: opts.Sender, + rooms: opts.Rooms, + policy: opts.Policy, + minBackoff: opts.MinBackoff, + maxBackoff: opts.MaxBackoff, + log: opts.Logger, + now: opts.Now, + wake: make(chan struct{}, 1), + workers: map[string]*worker{}, + } +} + +// Start begins sending: the dispatcher, and a worker for every destination +// with something left in the queue from before. +func (q *Queue) Start(ctx context.Context) error { + q.mu.Lock() + if q.running { + q.mu.Unlock() + return nil + } + q.ctx, q.cancel = context.WithCancel(context.WithoutCancel(ctx)) + q.running = true + q.txnPrefix = strconv.FormatInt(q.now().UnixNano(), 36) + q.mu.Unlock() + + q.wg.Add(1) + go q.dispatch() + + destinations, err := q.store.QueuedDestinations(ctx) + if err != nil { + return fmt.Errorf("read the outbound queue: %w", err) + } + for _, d := range destinations { + q.kick(d) + } + q.log.Info("federation queue started", "destinations_with_backlog", len(destinations)) + return nil +} + +// Stop halts every send and waits for the workers to finish. What is +// queued stays queued. +func (q *Queue) Stop() { + q.mu.Lock() + if !q.running { + q.mu.Unlock() + return + } + q.running = false + q.cancel() + q.incoming = nil + q.mu.Unlock() + + q.wg.Wait() + + q.mu.Lock() + q.workers = map[string]*worker{} + q.mu.Unlock() + q.log.Info("federation queue stopped") +} + +// Running reports whether the queue is sending. +func (q *Queue) Running() bool { + q.mu.Lock() + defer q.mu.Unlock() + return q.running +} + +// EventPersisted implements roomserver.Notifier. It must not block, and +// does not: it appends and signals. +// +// While the queue is stopped events are dropped rather than held. That is +// what the federation switch promises — no remote traffic while it is off +// — and holding them would turn switching federation back on after a month +// into a month of events delivered in one burst. A remote server that +// cares about the room fetches what it missed when it next hears from it. +func (q *Queue) EventPersisted(_ context.Context, n roomserver.Notification) { + if n.Origin != "" && !n.Relay { + return + } + q.mu.Lock() + if !q.running { + q.mu.Unlock() + return + } + q.incoming = append(q.incoming, n) + q.mu.Unlock() + select { + case q.wake <- struct{}{}: + default: + } +} + +// dispatch turns persisted events into queue rows. +func (q *Queue) dispatch() { + defer q.wg.Done() + for { + select { + case <-q.ctx.Done(): + return + case <-q.wake: + } + q.mu.Lock() + batch := q.incoming + q.incoming = nil + q.mu.Unlock() + + for _, n := range batch { + if err := q.queueEvent(q.ctx, n); err != nil && q.ctx.Err() == nil { + q.log.Error("could not queue an event for federation", + "room_id", n.RoomID, "event_id", n.Event.EventID(), "error", err) + } + } + } +} + +// queueEvent writes one event's queue rows. +func (q *Queue) queueEvent(ctx context.Context, n roomserver.Notification) error { + servers, err := q.rooms.JoinedServers(ctx, n.RoomID) + if err != nil { + return err + } + // A membership event also goes to the server of the user it is about: + // a server whose last user was just kicked is no longer joined, and is + // exactly the one that needs to hear it. + if key, ok := n.Event.StateKey(); ok && n.Event.Type() == event.TypeMember { + if server, err := id.ServerNameOf(key); err == nil && !slices.Contains(servers, server) { + servers = append(servers, server) + } + } + + var items []storage.QueuedItem + for _, server := range servers { + if server == q.serverName || server == n.Origin || !q.policy.Allowed(server) { + continue + } + items = append(items, storage.QueuedItem{ + Destination: server, + Kind: storage.QueueKindPDU, + RoomID: n.RoomID, + EventID: n.Event.EventID(), + Payload: n.Event.JSON(), + QueuedAtMS: q.now().UnixMilli(), + }) + } + return q.enqueue(ctx, items) +} + +// QueueEDU queues an EDU for a set of destinations. +func (q *Queue) QueueEDU(ctx context.Context, destinations []string, edu federation.EDU) { + if !q.Running() { + return + } + payload, err := json.Marshal(edu) + if err != nil { + q.log.Error("could not encode an EDU", "type", edu.Type, "error", err) + return + } + var items []storage.QueuedItem + for _, server := range destinations { + if server == q.serverName || !q.policy.Allowed(server) { + continue + } + items = append(items, storage.QueuedItem{ + Destination: server, + Kind: storage.QueueKindEDU, + Payload: payload, + QueuedAtMS: q.now().UnixMilli(), + }) + } + if err := q.enqueue(ctx, items); err != nil { + q.log.Error("could not queue an EDU", "type", edu.Type, "error", err) + } +} + +// enqueue writes rows and wakes their workers. +func (q *Queue) enqueue(ctx context.Context, items []storage.QueuedItem) error { + if len(items) == 0 { + return nil + } + if _, err := q.store.Enqueue(ctx, items); err != nil { + return err + } + seen := map[string]bool{} + for _, item := range items { + if !seen[item.Destination] { + seen[item.Destination] = true + q.kick(item.Destination) + } + } + return nil +} + +// kick wakes a destination's worker, starting one if there is none. +func (q *Queue) kick(destination string) { + q.mu.Lock() + defer q.mu.Unlock() + if !q.running { + return + } + w, ok := q.workers[destination] + if !ok { + w = &worker{destination: destination, wake: make(chan struct{}, 1)} + q.workers[destination] = w + q.wg.Add(1) + go q.run(w) + } + select { + case w.wake <- struct{}{}: + default: + } +} + +// run is one destination's worker. It sends until the queue for the +// destination is empty, then exits; the next enqueue starts a new one. +func (q *Queue) run(w *worker) { + defer q.wg.Done() + for { + dest, err := q.store.Destination(q.ctx, w.destination) + if err != nil { + if q.ctx.Err() != nil { + return + } + q.log.Error("could not read a destination's backoff", "destination", w.destination, "error", err) + dest = storage.Destination{ServerName: w.destination} + } + if wait := time.Duration(dest.RetryAtMS-q.now().UnixMilli()) * time.Millisecond; wait > 0 { + // A wake-up during the wait re-reads the backoff, which is how + // an administrator's reset takes effect at once; an ordinary + // enqueue finds the backoff unchanged and waits again. + timer := time.NewTimer(wait) + select { + case <-q.ctx.Done(): + timer.Stop() + return + case <-w.wake: + timer.Stop() + continue + case <-timer.C: + } + } + + sent, err := q.sendOnce(w.destination, dest) + if q.ctx.Err() != nil { + return + } + if err != nil { + // A failed send has recorded its backoff, which the next pass + // waits out. The pause here is for the other kind of failure — + // the queue's own database — which records nothing and would + // otherwise be retried in a tight loop. + select { + case <-q.ctx.Done(): + return + case <-time.After(time.Second): + } + continue + } + if !sent && q.retire(w) { + return + } + } +} + +// retire removes an idle worker, unless something was queued for it in +// the meantime. +func (q *Queue) retire(w *worker) bool { + q.mu.Lock() + defer q.mu.Unlock() + select { + case <-w.wake: + return false + default: + } + if q.workers[w.destination] == w { + delete(q.workers, w.destination) + } + return true +} + +// sendOnce sends one transaction. It reports whether there was anything to +// send. +func (q *Queue) sendOnce(destination string, health storage.Destination) (bool, error) { + pdus, err := q.store.Queued(q.ctx, destination, storage.QueueKindPDU, maxPDUsPerTxn) + if err != nil { + return false, err + } + edus, err := q.store.Queued(q.ctx, destination, storage.QueueKindEDU, maxEDUsPerTxn) + if err != nil { + return false, err + } + if len(pdus) == 0 && len(edus) == 0 { + return false, nil + } + + // A destination the policy no longer allows gets nothing, and what + // was queued for it is dropped: the allow and deny lists can change at + // runtime, and a backlog for a server that was just denied is exactly + // what must not go out. + if !q.policy.Allowed(destination) { + return true, q.store.Dequeue(q.ctx, append(itemIDs(pdus), itemIDs(edus)...)) + } + + txn := federation.Transaction{ + Origin: q.serverName, + OriginServerTS: q.now().UnixMilli(), + PDUs: make([]json.RawMessage, 0, len(pdus)), + } + for _, item := range pdus { + txn.PDUs = append(txn.PDUs, item.Payload) + } + for _, item := range edus { + var edu federation.EDU + if err := json.Unmarshal(item.Payload, &edu); err == nil { + txn.EDUs = append(txn.EDUs, edu) + } + } + + ctx, cancel := context.WithTimeout(q.ctx, sendTimeout) + defer cancel() + resp, err := q.sender.SendTransaction(ctx, destination, q.nextTxnID(), txn) + if err != nil { + if q.ctx.Err() != nil { + return true, err + } + q.recordFailure(destination, health, err) + return true, err + } + + for eventID, result := range resp.PDUs { + if result.Error != "" { + q.log.Info("a destination refused an event", + "destination", destination, "event_id", eventID, "error", result.Error) + } + } + if err := q.store.Dequeue(q.ctx, append(itemIDs(pdus), itemIDs(edus)...)); err != nil { + return true, err + } + if health.Failures > 0 || health.LastSuccessMS == 0 { + q.log.Info("destination is reachable", "destination", destination, "after_failures", health.Failures) + if err := q.store.SetDestination(q.ctx, storage.Destination{ + ServerName: destination, + LastSuccessMS: q.now().UnixMilli(), + LastFailureMS: health.LastFailureMS, + LastError: health.LastError, + }); err != nil { + q.log.Warn("could not record a destination's recovery", "destination", destination, "error", err) + } + } + return true, nil +} + +// recordFailure backs a destination off. +func (q *Queue) recordFailure(destination string, health storage.Destination, err error) { + failures := health.Failures + 1 + backoff := q.backoff(failures) + now := q.now() + q.log.Info("sending to a destination failed", + "destination", destination, "failures", failures, "retry_in", backoff, "error", err) + if serr := q.store.SetDestination(q.ctx, storage.Destination{ + ServerName: destination, + Failures: failures, + RetryAtMS: now.Add(backoff).UnixMilli(), + LastSuccessMS: health.LastSuccessMS, + LastFailureMS: now.UnixMilli(), + LastError: truncate(err.Error(), 500), + }); serr != nil { + q.log.Warn("could not record a destination's backoff", "destination", destination, "error", serr) + } +} + +// backoff is the wait after a number of consecutive failures: doubling +// from the minimum up to the cap, with a fifth of jitter so that a server +// coming back is not hit by every other server's retry at once. +func (q *Queue) backoff(failures int) time.Duration { + d := q.minBackoff + for i := 1; i < failures && d < q.maxBackoff; i++ { + d *= 2 + } + if d > q.maxBackoff { + d = q.maxBackoff + } + jitter := time.Duration(rand.Int64N(int64(d)/5 + 1)) + return d + jitter +} + +// nextTxnID mints a transaction ID. +func (q *Queue) nextTxnID() string { + q.mu.Lock() + defer q.mu.Unlock() + q.txnSeq++ + return q.txnPrefix + "_" + strconv.FormatUint(q.txnSeq, 10) +} + +// Destinations lists every destination the queue knows, for the admin API. +func (q *Queue) Destinations(ctx context.Context) ([]storage.DestinationSummary, error) { + return q.store.Destinations(ctx) +} + +// ResetBackoff clears a destination's backoff so that it is tried now, +// for an administrator who knows it is back. +func (q *Queue) ResetBackoff(ctx context.Context, destination string) error { + health, err := q.store.Destination(ctx, destination) + if err != nil { + return err + } + health.Failures = 0 + health.RetryAtMS = 0 + if err := q.store.SetDestination(ctx, health); err != nil { + return err + } + q.kick(destination) + return nil +} + +func itemIDs(items []storage.QueuedItem) []int64 { + out := make([]int64, 0, len(items)) + for _, item := range items { + out = append(out, item.ID) + } + return out +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} diff --git a/server/internal/federationapi/queue_test.go b/server/internal/federationapi/queue_test.go new file mode 100644 index 0000000..79312d3 --- /dev/null +++ b/server/internal/federationapi/queue_test.go @@ -0,0 +1,316 @@ +package federationapi + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "path/filepath" + "slices" + "sync" + "testing" + "time" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" + _ "github.com/dariuszjaniak/warp/server/internal/storage/drivers" +) + +const ourName = "warp.example" + +// fakeSender records transactions and fails on demand. +type fakeSender struct { + mu sync.Mutex + txns map[string][]federation.Transaction + ids map[string]bool + down map[string]bool +} + +func newFakeSender() *fakeSender { + return &fakeSender{txns: map[string][]federation.Transaction{}, ids: map[string]bool{}, down: map[string]bool{}} +} + +func (f *fakeSender) SendTransaction(_ context.Context, destination, txnID string, txn federation.Transaction) (federation.SendTransactionResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.down[destination] { + return federation.SendTransactionResponse{}, errors.New("connection refused") + } + if f.ids[txnID] { + return federation.SendTransactionResponse{}, fmt.Errorf("transaction ID %s reused", txnID) + } + f.ids[txnID] = true + f.txns[destination] = append(f.txns[destination], txn) + return federation.SendTransactionResponse{PDUs: map[string]federation.PDUResult{}}, nil +} + +func (f *fakeSender) setDown(destination string, down bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.down[destination] = down +} + +// pdus returns how many PDUs a destination has received. +func (f *fakeSender) pdus(destination string) int { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + for _, txn := range f.txns[destination] { + n += len(txn.PDUs) + } + return n +} + +func (f *fakeSender) largestTxn(destination string) int { + f.mu.Lock() + defer f.mu.Unlock() + largest := 0 + for _, txn := range f.txns[destination] { + largest = max(largest, len(txn.PDUs)) + } + return largest +} + +func (f *fakeSender) edus(destination string) []federation.EDU { + f.mu.Lock() + defer f.mu.Unlock() + var out []federation.EDU + for _, txn := range f.txns[destination] { + out = append(out, txn.EDUs...) + } + return out +} + +// fakeRooms answers JoinedServers from a table. +type fakeRooms map[string][]string + +func (f fakeRooms) JoinedServers(_ context.Context, roomID string) ([]string, error) { + return f[roomID], nil +} + +func openStore(t *testing.T) storage.FederationStore { + t.Helper() + db, err := storage.Open(t.Context(), storage.Config{ + Driver: storage.DriverSQLite, + ConnectionString: "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "q.db")) + "?_pragma=busy_timeout(5000)", + }, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + if err := db.Migrate(t.Context()); err != nil { + t.Fatal(err) + } + return db.Federation() +} + +func newTestQueue(t *testing.T, store storage.FederationStore, sender Sender, rooms RoomServers, policy *federation.Policy) *Queue { + t.Helper() + q := NewQueue(QueueOptions{ + ServerName: ourName, + Store: store, + Sender: sender, + Rooms: rooms, + Policy: policy, + MinBackoff: 20 * time.Millisecond, + MaxBackoff: 100 * time.Millisecond, + Logger: slog.New(slog.DiscardHandler), + }) + t.Cleanup(q.Stop) + return q +} + +var testKey = func() *signing.KeyPair { + k, err := signing.Generate("q") + if err != nil { + panic(err) + } + return k +}() + +// notification builds a persisted-event notification for a message, or a +// membership event about target when target is set. +func notification(t *testing.T, roomID, sender, target string, n int) roomserver.Notification { + t.Helper() + b := event.Builder{ + RoomID: roomID, Type: "m.room.message", Sender: sender, + Content: json.RawMessage(fmt.Sprintf(`{"body":"%d"}`, n)), + PrevEvents: []string{"$prev"}, AuthEvents: []string{"$auth"}, Depth: int64(n + 2), + } + if target != "" { + b.Type = event.TypeMember + b.StateKey = &target + b.Content = json.RawMessage(`{"membership":"leave"}`) + } + e, err := b.Build(time.Unix(1_700_000_000+int64(n), 0), testKey, roomversion.MustGet("11")) + if err != nil { + t.Fatal(err) + } + return roomserver.Notification{RoomID: roomID, Event: e, StreamOrdering: int64(n)} +} + +func waitFor(t *testing.T, what string, check func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if check() { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +func TestQueueSendsToTheRoomsServers(t *testing.T) { + sender := newFakeSender() + rooms := fakeRooms{"!r:warp.example": {ourName, "a.example", "b.example"}} + q := newTestQueue(t, openStore(t), sender, rooms, nil) + if err := q.Start(t.Context()); err != nil { + t.Fatal(err) + } + + q.EventPersisted(t.Context(), notification(t, "!r:warp.example", "@me:"+ourName, "", 1)) + // Received from b and not to be relayed: b sends its own events. + received := notification(t, "!r:warp.example", "@x:b.example", "", 2) + received.Origin = "b.example" + q.EventPersisted(t.Context(), received) + // A join handed over by send_join from c: relayed to everybody but c. + relayed := notification(t, "!r:warp.example", "@y:c.example", "", 3) + relayed.Origin, relayed.Relay = "c.example", true + q.EventPersisted(t.Context(), relayed) + // A kick of a user on d, whose server is no longer joined but must + // hear of it. + q.EventPersisted(t.Context(), notification(t, "!r:warp.example", "@me:"+ourName, "@z:d.example", 4)) + + waitFor(t, "deliveries", func() bool { + return sender.pdus("a.example") == 3 && sender.pdus("b.example") == 3 && sender.pdus("d.example") == 1 + }) + if n := sender.pdus("c.example"); n != 0 { + t.Errorf("c received %d PDUs; it is not in the room and its own join was not to be echoed", n) + } + if n := sender.pdus(ourName); n != 0 { + t.Errorf("the queue sent %d PDUs to this server", n) + } +} + +func TestQueueBacksOffAndRetries(t *testing.T) { + sender := newFakeSender() + sender.setDown("a.example", true) + store := openStore(t) + q := newTestQueue(t, store, sender, fakeRooms{"!r:warp.example": {"a.example"}}, nil) + if err := q.Start(t.Context()); err != nil { + t.Fatal(err) + } + + for i := range 120 { + q.EventPersisted(t.Context(), notification(t, "!r:warp.example", "@me:"+ourName, "", i)) + } + waitFor(t, "a recorded failure", func() bool { + d, err := store.Destination(t.Context(), "a.example") + return err == nil && d.Failures >= 2 + }) + + sender.setDown("a.example", false) + waitFor(t, "every PDU delivered", func() bool { return sender.pdus("a.example") == 120 }) + if largest := sender.largestTxn("a.example"); largest > maxPDUsPerTxn { + t.Errorf("a transaction carried %d PDUs, the limit is %d", largest, maxPDUsPerTxn) + } + waitFor(t, "the failure count to clear", func() bool { + d, err := store.Destination(t.Context(), "a.example") + return err == nil && d.Failures == 0 && d.LastSuccessMS > 0 + }) + left, err := store.Queued(t.Context(), "a.example", storage.QueueKindPDU, 10) + if err != nil || len(left) != 0 { + t.Errorf("queue after delivery = %d items, %v", len(left), err) + } +} + +func TestQueueSurvivesARestart(t *testing.T) { + sender := newFakeSender() + sender.setDown("a.example", true) + store := openStore(t) + rooms := fakeRooms{"!r:warp.example": {"a.example"}} + + first := newTestQueue(t, store, sender, rooms, nil) + if err := first.Start(t.Context()); err != nil { + t.Fatal(err) + } + for i := range 5 { + first.EventPersisted(t.Context(), notification(t, "!r:warp.example", "@me:"+ourName, "", i)) + } + waitFor(t, "the events to be queued", func() bool { + items, _ := store.Queued(t.Context(), "a.example", storage.QueueKindPDU, 10) + return len(items) == 5 + }) + first.Stop() + + // While stopped, events are dropped rather than held. + first.EventPersisted(t.Context(), notification(t, "!r:warp.example", "@me:"+ourName, "", 99)) + + sender.setDown("a.example", false) + if err := store.SetDestination(t.Context(), storage.Destination{ServerName: "a.example"}); err != nil { + t.Fatal(err) + } + second := newTestQueue(t, store, sender, rooms, nil) + if err := second.Start(t.Context()); err != nil { + t.Fatal(err) + } + waitFor(t, "the backlog after a restart", func() bool { return sender.pdus("a.example") == 5 }) + time.Sleep(50 * time.Millisecond) + if n := sender.pdus("a.example"); n != 5 { + t.Errorf("a received %d PDUs; the one persisted while stopped should not have been queued", n) + } +} + +func TestQueueHonoursThePolicy(t *testing.T) { + sender := newFakeSender() + policy := federation.NewPolicy(nil, []string{"denied.example"}) + q := newTestQueue(t, openStore(t), sender, fakeRooms{"!r:warp.example": {"ok.example", "denied.example"}}, policy) + if err := q.Start(t.Context()); err != nil { + t.Fatal(err) + } + q.EventPersisted(t.Context(), notification(t, "!r:warp.example", "@me:"+ourName, "", 1)) + waitFor(t, "delivery to the allowed server", func() bool { return sender.pdus("ok.example") == 1 }) + if n := sender.pdus("denied.example"); n != 0 { + t.Errorf("a denied server received %d PDUs", n) + } +} + +func TestEDUsGoToTheRoomsServers(t *testing.T) { + sender := newFakeSender() + rooms := fakeRooms{"!r:warp.example": {ourName, "a.example"}} + q := newTestQueue(t, openStore(t), sender, rooms, nil) + if err := q.Start(t.Context()); err != nil { + t.Fatal(err) + } + edus := NewEDUs(q, rooms) + edus.Typing(t.Context(), "!r:warp.example", "@me:"+ourName, true, 0) + edus.Receipt(t.Context(), "!r:warp.example", "@me:"+ourName, "m.read", "$e", "", 5) + edus.Receipt(t.Context(), "!r:warp.example", "@me:"+ourName, "m.read.private", "$e", "", 5) + edus.ToDevice(t.Context(), "@me:"+ourName, "m.room_key", map[string]map[string]json.RawMessage{ + "@u:a.example": {"DEV": json.RawMessage(`{"k":1}`)}, + "@v:b.example": {"*": json.RawMessage(`{"k":2}`)}, + }) + + waitFor(t, "EDUs", func() bool { return len(sender.edus("a.example")) == 3 && len(sender.edus("b.example")) == 1 }) + var types []string + for _, e := range sender.edus("a.example") { + types = append(types, e.Type) + } + slices.Sort(types) + if !slices.Equal(types, []string{federation.EDUDirectToDev, federation.EDUReceipt, federation.EDUTyping}) { + t.Errorf("EDUs to a = %v; a private receipt must not leave the server", types) + } + var toDevice federation.DirectToDeviceContent + if err := json.Unmarshal(sender.edus("b.example")[0].Content, &toDevice); err != nil { + t.Fatal(err) + } + if _, ok := toDevice.Messages["@u:a.example"]; ok { + t.Error("b was sent a's recipients") + } +} diff --git a/server/internal/federationapi/server.go b/server/internal/federationapi/server.go new file mode 100644 index 0000000..34302a9 --- /dev/null +++ b/server/internal/federationapi/server.go @@ -0,0 +1,661 @@ +package federationapi + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "strconv" + "strings" + "time" + + "github.com/dariuszjaniak/warp/server/internal/buildinfo" + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/httputil" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" + "github.com/dariuszjaniak/warp/server/internal/syncapi" +) + +// maxBody caps a federation request body. A transaction is up to 50 PDUs +// of up to 64 KiB each, plus EDUs; anything past this is not a +// transaction. +const maxBody = 8 << 20 + +// Error codes the federation API uses beyond the Client-Server set. +const ( + errCodeIncompatibleRoomVersion = "M_INCOMPATIBLE_ROOM_VERSION" + errCodeUnableToAuthoriseJoin = "M_UNABLE_TO_AUTHORISE_JOIN" +) + +// EphemeralSink takes in EDUs from other servers. *syncapi.API satisfies +// it. +type EphemeralSink interface { + SetTyping(ctx context.Context, req syncapi.TypingRequest) error + SetReceipt(ctx context.Context, req syncapi.ReceiptRequest) error + SendToDevice(ctx context.Context, req syncapi.SendToDeviceRequest) error +} + +// Profiles answers profile queries for local users. *userapi.API +// satisfies it. +type Profiles interface { + Account(ctx context.Context, localpart string) (storage.Account, error) +} + +// ServerOptions configure the inbound API. +type ServerOptions struct { + ServerName string + Gate *federation.Gate + Keys *federation.KeyRing + Rooms *roomserver.API + Ephemeral EphemeralSink + Profiles Profiles + Store storage.FederationStore + Logger *slog.Logger + Now func() time.Time +} + +// Server is the inbound Server-Server API. +type Server struct { + serverName string + gate *federation.Gate + keys *federation.KeyRing + rooms *roomserver.API + ephemeral EphemeralSink + profiles Profiles + store storage.FederationStore + log *slog.Logger + now func() time.Time +} + +// NewServer builds the inbound API. +func NewServer(opts ServerOptions) (*Server, error) { + switch { + case opts.ServerName == "": + return nil, errors.New("federationapi: ServerName must not be empty") + case opts.Gate == nil, opts.Keys == nil, opts.Rooms == nil, opts.Store == nil: + return nil, errors.New("federationapi: Gate, Keys, Rooms and Store are required") + } + if opts.Logger == nil { + opts.Logger = slog.Default() + } + if opts.Now == nil { + opts.Now = time.Now + } + return &Server{ + serverName: opts.ServerName, + gate: opts.Gate, + keys: opts.Keys, + rooms: opts.Rooms, + ephemeral: opts.Ephemeral, + profiles: opts.Profiles, + store: opts.Store, + log: opts.Logger, + now: opts.Now, + }, nil +} + +// authedHandler is a federation handler that runs after the request has +// been authenticated, with the origin and the body it was signed over. +type authedHandler func(w http.ResponseWriter, r *http.Request, origin string, body []byte) error + +// RegisterRoutes adds every federation endpoint, each behind the gate: with +// the switch off the whole surface answers M_UNRECOGNIZED, as a server +// with no federation API would. +func (s *Server) RegisterRoutes(r *httputil.Router) { + public := func(pattern string, h httputil.Handler) { + r.Handle(pattern, s.gate.ProtectFunc(h)) + } + authed := func(pattern string, h authedHandler) { + r.Handle(pattern, s.gate.ProtectFunc(s.authenticated(h))) + } + + // Keys and version need no authentication: a server has to be able to + // fetch our key before it can check anything we sign. + public("GET /_matrix/key/v2/server", s.handleServerKey) + public("GET /_matrix/key/v2/server/{keyID}", s.handleServerKey) + public("POST /_matrix/key/v2/query", s.handleKeyQuery) + public("GET /_matrix/key/v2/query/{serverName}", s.handleKeyQueryOne) + public("GET /_matrix/key/v2/query/{serverName}/{keyID}", s.handleKeyQueryOne) + public("GET /_matrix/federation/v1/version", s.handleVersion) + + const v1, v2 = "/_matrix/federation/v1", "/_matrix/federation/v2" + authed("PUT "+v1+"/send/{txnID}", s.handleSend) + authed("GET "+v1+"/make_join/{roomID}/{userID}", s.handleMakeJoin) + authed("PUT "+v1+"/send_join/{roomID}/{eventID}", s.handleSendJoinV1) + authed("PUT "+v2+"/send_join/{roomID}/{eventID}", s.handleSendJoinV2) + authed("GET "+v1+"/make_leave/{roomID}/{userID}", s.handleMakeLeave) + authed("PUT "+v1+"/send_leave/{roomID}/{eventID}", s.handleSendLeaveV1) + authed("PUT "+v2+"/send_leave/{roomID}/{eventID}", s.handleSendLeaveV2) + authed("PUT "+v2+"/invite/{roomID}/{eventID}", s.handleInvite) + authed("GET "+v1+"/event/{eventID}", s.handleEvent) + authed("GET "+v1+"/state/{roomID}", s.handleState) + authed("GET "+v1+"/state_ids/{roomID}", s.handleStateIDs) + authed("GET "+v1+"/event_auth/{roomID}/{eventID}", s.handleEventAuth) + authed("GET "+v1+"/backfill/{roomID}", s.handleBackfill) + authed("POST "+v1+"/get_missing_events/{roomID}", s.handleMissingEvents) + authed("GET "+v1+"/query/directory", s.handleQueryDirectory) + authed("GET "+v1+"/query/profile", s.handleQueryProfile) + authed("GET "+v1+"/publicRooms", s.handlePublicRoomsGet) + authed("POST "+v1+"/publicRooms", s.handlePublicRoomsPost) +} + +// authenticated checks X-Matrix and the allow/deny lists before the +// handler runs. +func (s *Server) authenticated(h authedHandler) httputil.Handler { + return func(w http.ResponseWriter, r *http.Request) error { + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxBody)) + if err != nil { + return httputil.NewError(http.StatusRequestEntityTooLarge, httputil.ErrCodeTooLarge, "Request body too large") + } + origin, err := s.keys.AuthenticateRequest(r.Context(), r, body) + if err != nil { + httputil.LoggerFrom(r.Context()).Info("federation request failed authentication", "error", err) + return httputil.NewError(http.StatusUnauthorized, httputil.ErrCodeUnauthorized, "%s", err.Error()) + } + if !s.gate.Policy().Allowed(origin) { + return httputil.Forbidden("Federation with %s is not allowed by this server", origin) + } + return h(w, r, origin, body) + } +} + +// fedError maps a roomserver error onto the Matrix error a remote server +// is sent. +func fedError(err error) error { + var matrixErr *httputil.MatrixError + switch { + case err == nil: + return nil + case errors.As(err, &matrixErr): + return err + case errors.Is(err, roomserver.ErrIncompatibleRoomVersion): + return httputil.NewError(http.StatusBadRequest, errCodeIncompatibleRoomVersion, "%s", err.Error()) + case errors.Is(err, roomserver.ErrNoSuchRoom), errors.Is(err, roomserver.ErrNoSuchEvent), + errors.Is(err, roomserver.ErrNoSuchAlias): + return httputil.NotFound("%s", err.Error()) + case errors.Is(err, roomserver.ErrForbidden), errors.Is(err, roomserver.ErrRejected), + errors.Is(err, roomserver.ErrNotInRoom): + return httputil.Forbidden("%s", err.Error()) + case errors.Is(err, roomserver.ErrBadEvent), errors.Is(err, roomserver.ErrInvalidContent): + return httputil.BadJSON("%s", err.Error()) + case errors.Is(err, roomserver.ErrFederationDisabled): + return httputil.Unrecognized() + case errors.Is(err, roomserver.ErrRemote): + return httputil.NewError(http.StatusBadGateway, httputil.ErrCodeUnknown, "%s", err.Error()) + } + return err +} + +// decode reads an already-read body. +func decode(body []byte, v any) error { + if len(body) == 0 { + return httputil.NotJSON() + } + if err := json.Unmarshal(body, v); err != nil { + return httputil.BadJSON("Invalid request body: %v", err) + } + return nil +} + +// --- keys and version ------------------------------------------------------- + +func (s *Server) handleServerKey(w http.ResponseWriter, _ *http.Request) error { + doc, err := s.keys.OwnKeyResponse() + if err != nil { + return err + } + httputil.WriteRawJSON(w, http.StatusOK, doc) + return nil +} + +// keyQueryResponse is the notary answer. +type keyQueryResponse struct { + ServerKeys []json.RawMessage `json:"server_keys"` +} + +func (s *Server) handleKeyQuery(w http.ResponseWriter, r *http.Request) error { + var req struct { + ServerKeys map[string]json.RawMessage `json:"server_keys"` + } + if err := httputil.DecodeJSON(w, r, &req); err != nil { + return err + } + resp := keyQueryResponse{ServerKeys: []json.RawMessage{}} + for serverName := range req.ServerKeys { + docs, err := s.keys.NotaryResponse(serverName) + if err != nil { + return err + } + resp.ServerKeys = append(resp.ServerKeys, docs...) + } + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} + +func (s *Server) handleKeyQueryOne(w http.ResponseWriter, r *http.Request) error { + docs, err := s.keys.NotaryResponse(r.PathValue("serverName")) + if err != nil { + return err + } + if docs == nil { + docs = []json.RawMessage{} + } + httputil.WriteJSON(w, http.StatusOK, keyQueryResponse{ServerKeys: docs}) + return nil +} + +func (s *Server) handleVersion(w http.ResponseWriter, _ *http.Request) error { + var resp federation.VersionResponse + resp.Server.Name = "Warp" + resp.Server.Version = buildinfo.Version + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} + +// --- transactions ------------------------------------------------------------- + +func (s *Server) handleSend(w http.ResponseWriter, r *http.Request, origin string, body []byte) error { + txnID := r.PathValue("txnID") + if cached, ok, err := s.store.InboundTransaction(r.Context(), origin, txnID); err != nil { + return err + } else if ok { + httputil.WriteRawJSON(w, http.StatusOK, cached) + return nil + } + + var txn federation.Transaction + if err := decode(body, &txn); err != nil { + return err + } + if txn.Origin != "" && txn.Origin != origin { + return httputil.Forbidden("The transaction's origin is %s, but it was signed by %s", txn.Origin, origin) + } + if len(txn.PDUs) > maxPDUsPerTxn || len(txn.EDUs) > maxEDUsPerTxn { + return httputil.BadJSON("A transaction carries at most %d PDUs and %d EDUs", maxPDUsPerTxn, maxEDUsPerTxn) + } + + resp := federation.SendTransactionResponse{PDUs: map[string]federation.PDUResult{}} + for _, raw := range txn.PDUs { + eventID, err := s.rooms.ReceivePDU(r.Context(), origin, raw) + if eventID == "" { + // Unparseable: there is no ID to report a result under. + s.log.Info("dropping an unreadable PDU", "origin", origin, "error", err) + continue + } + result := federation.PDUResult{} + if err != nil { + result.Error = err.Error() + s.log.Info("refused a PDU", "origin", origin, "event_id", eventID, "error", err) + } + resp.PDUs[eventID] = result + } + for _, edu := range txn.EDUs { + s.receiveEDU(r.Context(), origin, edu) + } + + encoded, err := json.Marshal(resp) + if err != nil { + return err + } + if err := s.store.RecordInboundTransaction(r.Context(), origin, txnID, encoded, s.now().UnixMilli()); err != nil { + s.log.Warn("could not record an inbound transaction", "origin", origin, "txn_id", txnID, "error", err) + } + httputil.WriteRawJSON(w, http.StatusOK, encoded) + return nil +} + +// sameServer reports whether a user belongs to the server that sent the +// EDU about them. Nothing in an EDU is signed, so this is the only check +// there is, and without it any server could make anybody appear to type. +func sameServer(userID, origin string) bool { + server, err := id.ServerNameOf(userID) + return err == nil && server == origin +} + +// receiveEDU takes one EDU in. Failures are logged and otherwise ignored: +// an EDU is ephemeral, and the transaction it came in succeeds whatever +// becomes of it. +func (s *Server) receiveEDU(ctx context.Context, origin string, edu federation.EDU) { + if s.ephemeral == nil { + return + } + switch edu.Type { + case federation.EDUTyping: + var c federation.TypingContent + if json.Unmarshal(edu.Content, &c) != nil || !sameServer(c.UserID, origin) || !s.allowed(ctx, c.RoomID, origin) { + return + } + if err := s.ephemeral.SetTyping(ctx, syncapi.TypingRequest{RoomID: c.RoomID, User: c.UserID, Typing: c.Typing}); err != nil { + s.log.Debug("ignored a typing EDU", "origin", origin, "room_id", c.RoomID, "error", err) + } + + case federation.EDUReceipt: + var c federation.ReceiptContent + if json.Unmarshal(edu.Content, &c) != nil { + return + } + for roomID, byType := range c { + if !s.allowed(ctx, roomID, origin) { + continue + } + for receiptType, byUser := range byType { + if receiptType != syncapi.ReceiptRead { + continue + } + for userID, data := range byUser { + if !sameServer(userID, origin) || len(data.EventIDs) == 0 { + continue + } + if err := s.ephemeral.SetReceipt(ctx, syncapi.ReceiptRequest{ + RoomID: roomID, User: userID, ReceiptType: receiptType, + EventID: data.EventIDs[len(data.EventIDs)-1], ThreadID: data.Data.ThreadID, + }); err != nil { + s.log.Debug("ignored a receipt EDU", "origin", origin, "room_id", roomID, "error", err) + } + } + } + } + + case federation.EDUDirectToDev: + var c federation.DirectToDeviceContent + if json.Unmarshal(edu.Content, &c) != nil || !sameServer(c.Sender, origin) { + return + } + if err := s.ephemeral.SendToDevice(ctx, syncapi.SendToDeviceRequest{ + Sender: c.Sender, EventType: c.Type, Messages: c.Messages, + }); err != nil { + s.log.Debug("ignored a to-device EDU", "origin", origin, "error", err) + } + + default: + // Presence and device list updates belong to phases that have not + // landed (presence is off by default; device lists are Phase 7). + s.log.Debug("ignored an EDU", "origin", origin, "type", edu.Type) + } +} + +// --- membership --------------------------------------------------------------- + +func (s *Server) handleMakeJoin(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error { + version, tmpl, err := s.rooms.MakeJoin(r.Context(), origin, r.PathValue("roomID"), r.PathValue("userID"), + r.URL.Query()["ver"]) + if err != nil { + if errors.Is(err, roomserver.ErrForbidden) && strings.Contains(err.Error(), "authorise a restricted join") { + return httputil.NewError(http.StatusBadRequest, errCodeUnableToAuthoriseJoin, "%s", err.Error()) + } + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, federation.MakeJoinResponse{RoomVersion: version, Event: tmpl}) + return nil +} + +func (s *Server) sendJoin(r *http.Request, origin string, body []byte) (federation.SendJoinResponse, error) { + res, err := s.rooms.SendJoin(r.Context(), origin, r.PathValue("roomID"), r.PathValue("eventID"), body) + if err != nil { + return federation.SendJoinResponse{}, fedError(err) + } + return federation.SendJoinResponse{ + Origin: s.serverName, + State: res.State, + AuthChain: res.AuthChain, + Event: res.Event, + ServersInRoom: res.ServersInRoom, + }, nil +} + +func (s *Server) handleSendJoinV2(w http.ResponseWriter, r *http.Request, origin string, body []byte) error { + resp, err := s.sendJoin(r, origin, body) + if err != nil { + return err + } + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} + +// handleSendJoinV1 is the deprecated form, which wraps the same answer in +// a [200, body] pair. +func (s *Server) handleSendJoinV1(w http.ResponseWriter, r *http.Request, origin string, body []byte) error { + resp, err := s.sendJoin(r, origin, body) + if err != nil { + return err + } + httputil.WriteJSON(w, http.StatusOK, []any{http.StatusOK, resp}) + return nil +} + +func (s *Server) handleMakeLeave(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error { + version, tmpl, err := s.rooms.MakeLeave(r.Context(), origin, r.PathValue("roomID"), r.PathValue("userID")) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, federation.MakeJoinResponse{RoomVersion: version, Event: tmpl}) + return nil +} + +func (s *Server) handleSendLeaveV2(w http.ResponseWriter, r *http.Request, origin string, body []byte) error { + if err := s.rooms.SendLeave(r.Context(), origin, r.PathValue("roomID"), r.PathValue("eventID"), body); err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, struct{}{}) + return nil +} + +func (s *Server) handleSendLeaveV1(w http.ResponseWriter, r *http.Request, origin string, body []byte) error { + if err := s.rooms.SendLeave(r.Context(), origin, r.PathValue("roomID"), r.PathValue("eventID"), body); err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, []any{http.StatusOK, struct{}{}}) + return nil +} + +func (s *Server) handleInvite(w http.ResponseWriter, r *http.Request, origin string, body []byte) error { + var req federation.InviteRequest + if err := decode(body, &req); err != nil { + return err + } + signed, err := s.rooms.ReceiveInvite(r.Context(), origin, req.RoomVersion, req.Event, req.InviteRoomState) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, federation.InviteResponse{Event: signed}) + return nil +} + +// --- reads -------------------------------------------------------------------- + +func (s *Server) transaction(pdus []json.RawMessage) federation.Transaction { + if pdus == nil { + pdus = []json.RawMessage{} + } + return federation.Transaction{Origin: s.serverName, OriginServerTS: s.now().UnixMilli(), PDUs: pdus} +} + +func (s *Server) handleEvent(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error { + raw, err := s.rooms.EventForServer(r.Context(), origin, r.PathValue("eventID")) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, s.transaction([]json.RawMessage{raw})) + return nil +} + +func (s *Server) handleState(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error { + eventID := r.URL.Query().Get("event_id") + if eventID == "" { + return httputil.MissingParam("event_id is required") + } + state, chain, err := s.rooms.StateForServer(r.Context(), origin, r.PathValue("roomID"), eventID) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, federation.StateResponse{PDUs: state, AuthChain: chain}) + return nil +} + +func (s *Server) handleStateIDs(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error { + eventID := r.URL.Query().Get("event_id") + if eventID == "" { + return httputil.MissingParam("event_id is required") + } + state, chain, err := s.rooms.StateIDsForServer(r.Context(), origin, r.PathValue("roomID"), eventID) + if err != nil { + return fedError(err) + } + if state == nil { + state = []string{} + } + if chain == nil { + chain = []string{} + } + httputil.WriteJSON(w, http.StatusOK, federation.StateIDsResponse{PDUIDs: state, AuthChainIDs: chain}) + return nil +} + +func (s *Server) handleEventAuth(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error { + chain, err := s.rooms.EventAuthForServer(r.Context(), origin, r.PathValue("roomID"), r.PathValue("eventID")) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, federation.EventAuthResponse{AuthChain: chain}) + return nil +} + +func (s *Server) handleBackfill(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error { + query := r.URL.Query() + from := query["v"] + if len(from) == 0 { + return httputil.MissingParam("v is required") + } + limit, err := strconv.Atoi(query.Get("limit")) + if err != nil || limit <= 0 { + return httputil.InvalidParam("limit must be a positive integer") + } + events, err := s.rooms.BackfillForServer(r.Context(), origin, r.PathValue("roomID"), from, limit) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, s.transaction(events)) + return nil +} + +func (s *Server) handleMissingEvents(w http.ResponseWriter, r *http.Request, origin string, body []byte) error { + var req federation.MissingEventsRequest + if err := decode(body, &req); err != nil { + return err + } + events, err := s.rooms.MissingEventsForServer(r.Context(), origin, r.PathValue("roomID"), + req.EarliestEvents, req.LatestEvents, req.Limit, req.MinDepth) + if err != nil { + return fedError(err) + } + if events == nil { + events = []json.RawMessage{} + } + httputil.WriteJSON(w, http.StatusOK, federation.MissingEventsResponse{Events: events}) + return nil +} + +// --- queries ------------------------------------------------------------------ + +func (s *Server) handleQueryDirectory(w http.ResponseWriter, r *http.Request, _ string, _ []byte) error { + alias := r.URL.Query().Get("room_alias") + if alias == "" { + return httputil.MissingParam("room_alias is required") + } + resolved, err := s.rooms.QueryAliasForServer(r.Context(), alias) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, federation.DirectoryResponse{RoomID: resolved.RoomID, Servers: resolved.Servers}) + return nil +} + +func (s *Server) handleQueryProfile(w http.ResponseWriter, r *http.Request, _ string, _ []byte) error { + query := r.URL.Query() + userID := query.Get("user_id") + localpart, server, err := id.SplitUserID(userID) + if err != nil { + return httputil.InvalidParam("user_id is not a user ID") + } + if server != s.serverName || s.profiles == nil { + return httputil.NotFound("%s is not a user of this server", userID) + } + account, err := s.profiles.Account(r.Context(), localpart) + if errors.Is(err, storage.ErrNoSuchUser) { + return httputil.NotFound("Profile not found") + } else if err != nil { + return err + } + resp := map[string]string{} + switch field := query.Get("field"); field { + case "": + if account.DisplayName != "" { + resp["displayname"] = account.DisplayName + } + if account.AvatarURL != "" { + resp["avatar_url"] = account.AvatarURL + } + case "displayname": + if account.DisplayName != "" { + resp["displayname"] = account.DisplayName + } + case "avatar_url": + if account.AvatarURL != "" { + resp["avatar_url"] = account.AvatarURL + } + default: + return httputil.InvalidParam("field must be displayname or avatar_url") + } + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} + +func (s *Server) handlePublicRoomsGet(w http.ResponseWriter, r *http.Request, _ string, _ []byte) error { + query := r.URL.Query() + limit, _ := strconv.Atoi(query.Get("limit")) + return s.publicRooms(w, r, roomserver.PublicRoomsRequest{Limit: limit, Since: query.Get("since")}) +} + +func (s *Server) handlePublicRoomsPost(w http.ResponseWriter, r *http.Request, _ string, body []byte) error { + var req struct { + Limit int `json:"limit"` + Since string `json:"since"` + Filter struct { + GenericSearchTerm string `json:"generic_search_term"` + } `json:"filter"` + } + if len(body) > 0 { + if err := decode(body, &req); err != nil { + return err + } + } + return s.publicRooms(w, r, roomserver.PublicRoomsRequest{ + Limit: req.Limit, Since: req.Since, Search: req.Filter.GenericSearchTerm, + }) +} + +func (s *Server) publicRooms(w http.ResponseWriter, r *http.Request, req roomserver.PublicRoomsRequest) error { + resp, err := s.rooms.PublicRooms(r.Context(), req) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} + +// String is for logs. +func (s *Server) String() string { return fmt.Sprintf("federation API for %s", s.serverName) } + +// allowed applies a room's server ACL to an EDU about it. An EDU is not +// a room event, but it is still a server speaking in the room, and a +// server the room has banned does not get to make its users appear to +// type. +func (s *Server) allowed(ctx context.Context, roomID, origin string) bool { + ok, err := s.rooms.ServerAllowed(ctx, roomID, origin) + return err == nil && ok +} diff --git a/server/internal/matrix/event/event.go b/server/internal/matrix/event/event.go index 46eef71..3b38a33 100644 --- a/server/internal/matrix/event/event.go +++ b/server/internal/matrix/event/event.go @@ -19,8 +19,8 @@ // can. Warp does not: such an event is rejected whatever its room version. // It only arises for events that were already outside the current grammar // when they were created, and handling them means a second, lenient -// encoder — which can wait until federation (PLAN.md phase 5) makes old -// rooms reachable at all. +// encoder — which waits for a room that needs it: joining the rooms of +// today over federation (PLAN.md phase 5) does not. package event import ( diff --git a/server/internal/matrix/roomversion/roomversion.go b/server/internal/matrix/roomversion/roomversion.go index 8c9ae2e..b21204b 100644 --- a/server/internal/matrix/roomversion/roomversion.go +++ b/server/internal/matrix/roomversion/roomversion.go @@ -19,8 +19,8 @@ // prev_events and auth_events as [id, hashes] tuples rather than plain // lists, and use state resolution v1, which is enough of a separate event // format to be its own piece of work. It is only needed to join rooms -// created before 2018 on other servers, so it waits for federation -// (PLAN.md phase 5) and the registry rejects those rooms clearly until then. +// created before 2018 on other servers, which federation (PLAN.md phase 5) +// does not yet do; the registry rejects those rooms clearly until it does. package roomversion import ( diff --git a/server/internal/matrix/signing/key.go b/server/internal/matrix/signing/key.go index cd485af..e6f5416 100644 --- a/server/internal/matrix/signing/key.go +++ b/server/internal/matrix/signing/key.go @@ -130,7 +130,8 @@ func (k *KeyPair) Save(path string) error { // Load reads a key file written by Save. Lines starting with '#' and blank // lines are ignored; the first key line wins. Older keys (a file with -// several entries) will matter once key rotation lands in Phase 5. +// several entries) will matter once key rotation exists, which it does not +// yet: see the known gaps in docs/federation.md. func Load(path string) (*KeyPair, error) { data, err := os.ReadFile(path) if err != nil { diff --git a/server/internal/roomserver/acl.go b/server/internal/roomserver/acl.go new file mode 100644 index 0000000..473fc93 --- /dev/null +++ b/server/internal/roomserver/acl.go @@ -0,0 +1,138 @@ +package roomserver + +import ( + "context" + "encoding/json" + "fmt" + "net" + "regexp" + "strings" + "sync" + + "github.com/dariuszjaniak/warp/server/internal/federation" +) + +// TypeServerACL is the state event that says which servers may take part +// in a room. +const TypeServerACL = "m.room.server_acl" + +// Server ACLs. +// +// A room can ban whole servers. The ACL is ordinary room state, and what +// it does is decided by every server in the room for itself: requests and +// events from a server the ACL denies are refused, as if that server were +// not there. It is the one defence a room has against a server that +// creates users faster than moderators can ban them. +// +// It applies to what a server *sends*: its transactions' PDUs, its EDUs, +// its joins and leaves, and its reads of the room's history. It does not +// rewrite the past — events already in the room stay — and it does not +// apply to the server enforcing it. + +// aclContent is the content of an m.room.server_acl event. +type aclContent struct { + Allow []string `json:"allow"` + Deny []string `json:"deny"` + AllowIPLiterals *bool `json:"allow_ip_literals"` +} + +// globCache keeps compiled ACL globs: the same handful of patterns are +// matched against every request a busy room sees. +var globCache sync.Map // pattern -> *regexp.Regexp + +// globMatch matches a server name against an ACL glob, where '*' is any +// run of characters and '?' is any one. The match ignores case, as every +// implementation does: a DNS name is not case-sensitive, and an ACL that +// "example.com" got around by writing "EXAMPLE.com" would be no ACL. +func globMatch(pattern, name string) bool { + if cached, ok := globCache.Load(pattern); ok { + return cached.(*regexp.Regexp).MatchString(name) + } + var b strings.Builder + b.WriteString(`(?i)^`) + for _, r := range pattern { + switch r { + case '*': + b.WriteString(`.*`) + case '?': + b.WriteString(`.`) + default: + b.WriteString(regexp.QuoteMeta(string(r))) + } + } + b.WriteString(`$`) + re, err := regexp.Compile(b.String()) + if err != nil { + return false + } + globCache.Store(pattern, re) + return re.MatchString(name) +} + +// aclAllows applies one ACL to a server name. The port is not part of +// what an ACL matches. +func aclAllows(acl aclContent, serverName string) bool { + host := serverName + if parsed, err := federation.ParseServerName(serverName); err == nil { + host = parsed.Host + if parsed.IsIP && acl.AllowIPLiterals != nil && !*acl.AllowIPLiterals { + return false + } + } else if h, _, err := net.SplitHostPort(serverName); err == nil { + host = h + } + for _, deny := range acl.Deny { + if globMatch(deny, host) { + return false + } + } + for _, allow := range acl.Allow { + if globMatch(allow, host) { + return true + } + } + // An ACL with an empty allow list allows nobody. That is what the + // spec says, and the room's own server is protected from it only + // because a server never applies a room's ACL to itself. + return false +} + +// ServerAllowed reports whether a room's ACL lets a server take part. A +// room with no ACL allows everybody, and this server is always allowed. +func (r *API) ServerAllowed(ctx context.Context, roomID, serverName string) (bool, error) { + if serverName == r.serverName { + return true, nil + } + stored, err := r.store.CurrentStateEvent(ctx, roomID, TypeServerACL, "") + if isNotFound(err) { + return true, nil + } else if err != nil { + return false, err + } + acl, ok := parseACL(contentOf(stored)) + if !ok { + // An ACL nobody can read restricts nobody; refusing everyone over + // a malformed event would let one bad state event cut a room off. + r.log.Warn("ignoring an unreadable server ACL", "room_id", roomID, "event_id", stored.EventID) + return true, nil + } + return aclAllows(acl, serverName), nil +} + +// parseACL decodes an ACL's content, reporting whether it could. +func parseACL(content json.RawMessage) (aclContent, bool) { + var acl aclContent + return acl, json.Unmarshal(content, &acl) == nil +} + +// requireAllowed refuses a server the room's ACL denies. +func (r *API) requireAllowed(ctx context.Context, roomID, serverName string) error { + ok, err := r.ServerAllowed(ctx, roomID, serverName) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("%w: %s is denied by the server ACL of %s", ErrForbidden, serverName, roomID) + } + return nil +} diff --git a/server/internal/roomserver/acl_internal_test.go b/server/internal/roomserver/acl_internal_test.go new file mode 100644 index 0000000..429d34f --- /dev/null +++ b/server/internal/roomserver/acl_internal_test.go @@ -0,0 +1,41 @@ +package roomserver + +import "testing" + +func TestACLMatching(t *testing.T) { + no := false + acl := aclContent{ + Allow: []string{"*"}, + Deny: []string{"evil.example", "*.bad.example", "spam?.example"}, + AllowIPLiterals: &no, + } + cases := map[string]bool{ + "good.example": true, + "GOOD.example": true, + "evil.example": false, + "EVIL.EXAMPLE": false, + "evil.example:8448": false, // the port is not part of the match + "a.bad.example": false, + "bad.example": true, // "*.bad.example" needs a subdomain + "spam1.example": false, + "spam12.example": true, + "1.2.3.4": false, + "[::1]:8448": false, + "notevil.example": true, + "evil.example.attacker": true, + } + for server, want := range cases { + if got := aclAllows(acl, server); got != want { + t.Errorf("aclAllows(%q) = %v, want %v", server, got, want) + } + } + + // An empty allow list allows nobody. + if aclAllows(aclContent{}, "anyone.example") { + t.Error("an ACL with no allow list let a server in") + } + // IP literals are allowed unless the ACL says otherwise. + if !aclAllows(aclContent{Allow: []string{"*"}}, "1.2.3.4") { + t.Error("an IP literal was refused by an ACL that does not mention them") + } +} diff --git a/server/internal/roomserver/backfill.go b/server/internal/roomserver/backfill.go new file mode 100644 index 0000000..b751562 --- /dev/null +++ b/server/internal/roomserver/backfill.go @@ -0,0 +1,323 @@ +package roomserver + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "sync" + + "github.com/dariuszjaniak/warp/server/internal/matrix/auth" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/matrix/stateres" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Outbound backfill: asking other servers for a room's history. +// +// A room joined over federation starts, on this server, at the join. The +// join is handed the room's state, which is enough to take part, but none +// of the events that led to it — so a client scrolling up reaches the join +// and stops, in a room that may have years of history on every other +// server in it. Backfill is what fills that in: when /messages pages back +// past the oldest event held, the servers in the room are asked for what +// came before it (GET /backfill), and what they send is checked like any +// other event from another server and stored *below* everything already +// here. +// +// "Below" is the whole difficulty. The stream ordering is the order clients +// are shown a room in, and its counter only goes up, because every event it +// has ever numbered arrived after the ones before it. History arrives last +// and belongs first, so it is numbered by a second counter that runs +// downwards from -1 (storage.RoomStore.WriteBackfill). Nothing streams +// through those positions: a live /sync is "everything after my token", +// which never reaches below zero, so backfilled history reaches clients +// only through /messages, /context and the state at a point in it — which +// is right, because it is not news. +// +// A backfilled event goes through the same checks as one that arrived +// live, but one: it is not checked against the room's current state. Soft +// failure is about refusing events built on a state the room has since +// moved past, and history is by definition built on a state the room has +// since moved past. + +// Bounds on what one page of /messages can make this server fetch. +const ( + // backfillMinBatch is the fewest events one /backfill request asks + // for, however small the page. A request costs a round trip and a + // state fetch at its far end whatever its size, so asking for a + // handful at a time would make scrolling a room's history slow for no + // saving. + backfillMinBatch = 20 + + // backfillRounds is how many /backfill requests one page may make + // before it is returned short. Events a server sends that fail the + // checks, or that the page's filter leaves out, are why one round may + // not be enough. + backfillRounds = 3 + + // backfillFrontierScan is how many of the room's oldest events are + // read to find where its history stops. + backfillFrontierScan = 50 +) + +// batchEvent is an event of a backfill batch that has passed the checks, +// with the state after it: what the events after it in the batch are +// checked against. +type batchEvent struct { + event *event.Event + stateAfter []*event.Event +} + +// wantsBackfill reports whether a page of a room's history came back short +// at the bottom of what this server holds, where other servers may hold +// more. +func (r *API) wantsBackfill(req TimelineRequest, visible VisibleRange, page Timeline) bool { + if !req.Backfill || req.Direction != storage.DirectionBackward || page.Limited || !r.federating() { + return false + } + // Only a window that is open at the bottom — no `to` from the client, + // and a history visibility that lets the user see before their join — + // can reach history this server does not have. + if low, _, ok := visible.Clamp(req.Low, req.High); !ok || low != 0 { + return false + } + // A page that reaches the create event has reached the beginning of + // the room. + return len(page.Events) == 0 || page.Events[0].Type != event.TypeCreate +} + +// backfillLock serialises the backfills of one room, so that two clients +// scrolling the same room at once do not fetch the same history twice. +// It is not the room's write lock: a backfill spends most of its time +// waiting for other servers, and live events must not wait with it. +func (r *API) backfillLock(roomID string) *sync.Mutex { + actual, _ := r.backfills.LoadOrStore(roomID, &sync.Mutex{}) + mu, ok := actual.(*sync.Mutex) + if !ok { + // Unreachable: nothing else ever stores into this map. + panic("roomserver: backfill lock map holds something that is not a mutex") + } + return mu +} + +// backfill asks the servers in a room for the history before the oldest +// event this server holds, and stores what passes below it. It returns how +// many events were added; zero with no error means there was nothing to +// ask for, or nobody had anything. +func (r *API) backfill(ctx context.Context, room storage.Room, rules roomversion.Rules, want int) (int, error) { + mu := r.backfillLock(room.ID) + mu.Lock() + defer mu.Unlock() + + // Worked out under the lock: a backfill that finished while this one + // waited has moved it. + frontier, err := r.backfillFrontier(ctx, room.ID, rules) + if err != nil || len(frontier) == 0 { + return 0, err + } + limit := min(max(want, backfillMinBatch), maxBackfillLimit) + + var lastErr error + for _, server := range r.candidateServers(ctx, room.ID, nil) { + if err := r.requireFederation("backfilling a room"); err != nil { + return 0, err + } + txn, err := r.remote.Backfill(ctx, server, room.ID, frontier, limit) + if err != nil { + lastErr = remoteError(server, err) + r.log.Info("a server could not backfill a room", "room_id", room.ID, "server", server, "error", err) + continue + } + added, err := r.acceptBackfill(ctx, server, room, rules, txn.PDUs) + if err != nil { + return 0, err + } + if added > 0 { + return added, nil + } + // A server that joined after this one did may have none of the + // history; the next one may. + } + return 0, lastErr +} + +// backfillFrontier finds where a room's history stops on this server: the +// prev_events of its oldest timeline events that are not themselves in the +// timeline. They are what /backfill is asked to start from. +// +// The oldest events, not only the oldest one: a room's history is a graph, +// and a batch that ends partway down two branches leaves an edge on each. +// Reading a bounded number of them keeps this one query however long the +// room is; the edges it misses are the ones a gap left higher up, which +// are not history. +func (r *API) backfillFrontier(ctx context.Context, roomID string, rules roomversion.Rules) ([]string, error) { + oldest, err := r.store.Timeline(ctx, storage.TimelineRequest{ + RoomID: roomID, + Direction: storage.DirectionForward, + Limit: backfillFrontierScan, + }) + if err != nil { + return nil, err + } + inScan := make(map[string]bool, len(oldest)) + for _, s := range oldest { + inScan[s.EventID] = true + } + var prevs []string + for _, s := range oldest { + e, err := parseEvent(s, rules) + if err != nil { + return nil, err + } + for _, id := range e.PrevEventIDs() { + if !inScan[id] && !slices.Contains(prevs, id) { + prevs = append(prevs, id) + } + } + } + if len(prevs) == 0 { + return nil, nil + } + + // An outlier is held without its place in the timeline, so it is still + // history to ask for; an event in the timeline further up is not. + held, err := r.store.EventsByID(ctx, roomID, prevs) + if err != nil { + return nil, err + } + for _, s := range held { + if !s.Outlier { + prevs = slices.DeleteFunc(prevs, func(id string) bool { return id == s.EventID }) + } + } + return prevs, nil +} + +// acceptBackfill checks the events a /backfill answered with, oldest +// first, and stores the ones that pass in one write. +// +// Each goes through the intake's steps 1 to 3 (see federated.go): it +// verifies, it passes the rules against its own auth events, and it passes +// them against the state before it. That state comes from the events +// before it in the same batch where it can, and otherwise from the state +// at its prev_events as the sending server reports it — which, for the +// oldest events of a batch, is a /state_ids request: the price of having a +// state snapshot for every backfilled event, which is what lets a client +// be told the room's state at any point of its history. +// +// An event that fails is left out and logged. It is not an error: the rest +// of the batch may still be good, and a server that sends bad history is +// one this server will not take history from, not one that breaks +// /messages. +func (r *API) acceptBackfill(ctx context.Context, origin string, room storage.Room, rules roomversion.Rules, raws []json.RawMessage) (int, error) { + var events []*event.Event + seen := map[string]bool{} + for _, raw := range raws { + e, err := parseRemote(raw, rules) + if err != nil { + r.log.Debug("ignoring an unparseable backfilled event", "room_id", room.ID, "origin", origin, "error", err) + continue + } + if !inRoom(e, room.ID) || seen[e.EventID()] { + continue + } + seen[e.EventID()] = true + events = append(events, e) + } + + // What is already in the timeline needs nothing; an outlier is checked + // again, because it is about to be given a place in the timeline and + // a state snapshot, which it never had. + ids := make([]string, 0, len(events)) + for _, e := range events { + ids = append(ids, e.EventID()) + } + held, err := r.store.EventsByID(ctx, room.ID, ids) + if err != nil { + return 0, err + } + for _, s := range held { + if !s.Outlier { + events = slices.DeleteFunc(events, func(e *event.Event) bool { return e.EventID() == s.EventID }) + } + } + if len(events) == 0 { + return 0, nil + } + + eventsByDepth(events) + verified := r.verifyAll(ctx, events) + create, err := r.createEvent(ctx, room, rules) + if err != nil { + return 0, err + } + + batch := map[string]batchEvent{} + writes := make([]storage.BackfillEvent, 0, len(verified)) + rejected := 0 + for _, e := range verified { + in := incoming{origin: origin, room: room, rules: rules, event: e, batch: batch} + stateBefore, err := r.checkBackfilled(ctx, in, create) + if err != nil { + rejected++ + r.log.Debug("a backfilled event was not accepted", + "room_id", room.ID, "origin", origin, "event_id", e.EventID(), "type", e.Type(), "error", err) + continue + } + after := stateres.Sorted(stateBefore) + if e.IsState() { + after = append(after, e) + } + batch[e.EventID()] = batchEvent{event: e, stateAfter: after} + writes = append(writes, storage.BackfillEvent{ + Event: storedOf(room.ID, e), + StateBefore: stateEntries(stateBefore), + }) + } + if rejected > 0 || len(verified) < len(events) { + r.log.Info("refused events in a backfill", + "room_id", room.ID, "origin", origin, + "unverified", len(events)-len(verified), "rejected", rejected) + } + + if err := r.store.WriteBackfill(ctx, room.ID, writes); err != nil { + return 0, err + } + return len(writes), nil +} + +// checkBackfilled runs one backfilled event through the intake's checks and +// returns the state before it. +func (r *API) checkBackfilled(ctx context.Context, in incoming, create *event.Event) (auth.State, error) { + e := in.event + + authEvents, err := r.authEventsOf(ctx, in) + if err != nil { + return nil, err + } + if err := auth.CheckAuthEvents(e, authEvents); err != nil { + return nil, err + } + if err := auth.Check(ctx, e, withCreate(auth.NewState(authEvents), create), r.verifier()); err != nil { + return nil, err + } + + // The create event is the one event with nothing before it. + if e.Type() == event.TypeCreate && len(e.PrevEventIDs()) == 0 { + if create == nil || e.EventID() != create.EventID() { + return nil, fmt.Errorf("%w: a second create event", ErrBadEvent) + } + return auth.State{}, nil + } + + stateBefore, err := r.stateBeforeIncoming(ctx, in) + if err != nil { + return nil, err + } + if err := auth.Check(ctx, e, withCreate(stateBefore, create), r.verifier()); err != nil { + return nil, fmt.Errorf("%w (against the state before it)", err) + } + return stateBefore, nil +} diff --git a/server/internal/roomserver/backfill_test.go b/server/internal/roomserver/backfill_test.go new file mode 100644 index 0000000..a7bc425 --- /dev/null +++ b/server/internal/roomserver/backfill_test.go @@ -0,0 +1,222 @@ +package roomserver_test + +import ( + "context" + "fmt" + "slices" + "testing" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Outbound backfill: a server that joined a room part-way through filling +// in the history before its join when a client scrolls back to it. + +func (t transport) Backfill(ctx context.Context, dest, roomID string, from []string, limit int) (federation.Transaction, error) { + api, err := t.to(dest) + if err != nil { + return federation.Transaction{}, err + } + pdus, err := api.BackfillForServer(ctx, t.from.name, roomID, from, limit) + if err != nil { + return federation.Transaction{}, httpError(err) + } + return federation.Transaction{Origin: dest, PDUs: pdus}, nil +} + +// scrollBack pages a room's history backwards the way a client scrolling +// up does — /messages from the end, then from each page's `end` token — +// and returns every event ID it was shown, oldest first. +func (p *peer) scrollBack(t *testing.T, roomID, user string, pageSize int) []string { + t.Helper() + var ids []string + var high int64 + for range 100 { + page, err := p.api.Timeline(t.Context(), roomserver.TimelineRequest{ + RoomID: roomID, User: user, High: high, + Direction: storage.DirectionBackward, Limit: pageSize, Backfill: true, + }) + if err != nil { + t.Fatalf("%s: Timeline: %v", p.name, err) + } + page.Events = slices.Clone(page.Events) + slices.Reverse(page.Events) + for _, e := range page.Events { + ids = append(ids, e.EventID) + } + if !page.Limited { + slices.Reverse(ids) + return ids + } + high = page.Start + } + t.Fatal("scrolling back never reached the end of the room") + return nil +} + +func TestBackfillFillsInTheHistoryBeforeAJoin(t *testing.T) { + for _, version := range []string{"10", "11", "12"} { + t.Run("v"+version, func(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{ + Creator: userA, Preset: roomserver.PresetPublicChat, Name: "Federated", Version: version, + }) + if err != nil { + t.Fatalf("CreateRoom: %v", err) + } + roomID := resp.RoomID + + // Enough history that it takes several /backfill requests, + // and a state change in the middle of it. + for i := range 25 { + a.send(t, roomID, userA, fmt.Sprint("before ", i)) + } + topicID, err := a.api.SendEvent(t.Context(), roomserver.SendEventRequest{ + RoomID: roomID, Sender: userA, Type: roomserver.TypeTopic, StateKey: ptrTo(""), + Content: []byte(`{"topic":"history"}`), + }) + if err != nil { + t.Fatalf("SendEvent: %v", err) + } + for i := range 10 { + a.send(t, roomID, userA, fmt.Sprint("after the topic ", i)) + } + a.takeOutbox() // nobody else is in the room yet + + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{ + RoomID: roomID, User: userB, ServerNames: []string{serverA}, + }); err != nil { + t.Fatalf("remote join: %v", err) + } + a.send(t, roomID, userA, "after the join") + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + + // Before anybody scrolls, B's timeline starts at its join. + if held := b.timelineIDs(t, roomID); len(held) != 2 { + t.Fatalf("B's timeline before backfill = %d events, want the join and one message", len(held)) + } + top, err := b.api.StreamTop(t.Context()) + if err != nil { + t.Fatalf("StreamTop: %v", err) + } + + // Scrolling back on B shows the whole room, in A's order. + want := a.timelineIDs(t, roomID) + got := b.scrollBack(t, roomID, userB, 10) + if !slices.Equal(got, want) { + t.Fatalf("B's scrollback has %d events, A's room %d:\nB: %v\nA: %v", len(got), len(want), got, want) + } + + // The history is in B's timeline proper, below everything B + // held, and is not news: the top of the stream — what a live + // /sync reads from — has not moved. + create, err := b.store.Event(t.Context(), roomID, want[0]) + if err != nil { + t.Fatalf("Event(create): %v", err) + } + if create.Outlier || create.StreamOrdering >= 0 || create.Type != event.TypeCreate { + t.Errorf("B's create event: type %s, position %d, outlier %v; want a timeline event below zero", + create.Type, create.StreamOrdering, create.Outlier) + } + if after, err := b.api.StreamTop(t.Context()); err != nil { + t.Fatalf("StreamTop: %v", err) + } else if after != top { + t.Errorf("backfill moved the stream top from %d to %d", top, after) + } + + // The room's state at a point inside the history is the state + // then: no topic before the topic was set, the topic after. + topic, err := b.store.Event(t.Context(), roomID, topicID) + if err != nil { + t.Fatalf("Event(topic): %v", err) + } + for pos, wantTopic := range map[int64]bool{ + storage.PositionBefore(topic.StreamOrdering): false, + topic.StreamOrdering: true, + } { + state, err := b.api.StateAt(t.Context(), roomID, userB, pos) + if err != nil { + t.Fatalf("StateAt(%d): %v", pos, err) + } + hasTopic, hasName := false, false + for _, e := range state { + hasTopic = hasTopic || e.Type == roomserver.TypeTopic + hasName = hasName || e.Type == roomserver.TypeName + } + if hasTopic != wantTopic || !hasName { + t.Errorf("state at %d: topic %v, name %v; want topic %v and the name", pos, hasTopic, hasName, wantTopic) + } + } + + // Scrolling back again asks nobody anything and shows the same. + if again := b.scrollBack(t, roomID, userB, 7); !slices.Equal(again, want) { + t.Errorf("a second scrollback differs:\n%v\n%v", again, want) + } + }) + } +} + +func TestBackfillWithFederationOffShowsWhatIsHeld(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + for i := range 5 { + a.send(t, roomID, userA, fmt.Sprint(i)) + } + a.takeOutbox() + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{ + RoomID: roomID, User: userB, ServerNames: []string{serverA}, + }); err != nil { + t.Fatalf("remote join: %v", err) + } + + // With the switch off the page is what B has, and no error: the client + // is not told /messages failed because another server could not be + // asked. + m.setDisabled(true) + got := b.scrollBack(t, roomID, userB, 10) + if held := b.timelineIDs(t, roomID); !slices.Equal(got, held) { + t.Errorf("scrollback with federation off = %v, want what B holds: %v", got, held) + } + + // And once it is back on, the history is there. + m.setDisabled(false) + if got := b.scrollBack(t, roomID, userB, 10); !slices.Equal(got, a.timelineIDs(t, roomID)) { + t.Errorf("scrollback with federation back on = %d events, want A's %d", len(got), len(a.timelineIDs(t, roomID))) + } +} + +func TestBackfillIsNotAskedForUnderJoinedVisibility(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + if _, err := a.api.SendEvent(t.Context(), roomserver.SendEventRequest{ + RoomID: roomID, Sender: userA, Type: event.TypeHistoryVisibility, StateKey: ptrTo(""), + Content: []byte(`{"history_visibility":"joined"}`), + }); err != nil { + t.Fatalf("SendEvent: %v", err) + } + a.send(t, roomID, userA, "before") + a.takeOutbox() + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{ + RoomID: roomID, User: userB, ServerNames: []string{serverA}, + }); err != nil { + t.Fatalf("remote join: %v", err) + } + + // Bob may not see before his join, so there is nothing to fetch for + // him, and nothing is. + if got := b.scrollBack(t, roomID, userB, 10); len(got) != 1 { + t.Errorf("scrollback = %v, want just the join", got) + } + if held := b.timelineIDs(t, roomID); len(held) != 1 { + t.Errorf("B's timeline = %v, want nothing backfilled", held) + } +} diff --git a/server/internal/roomserver/directory.go b/server/internal/roomserver/directory.go index e7a5a8e..6ae0b48 100644 --- a/server/internal/roomserver/directory.go +++ b/server/internal/roomserver/directory.go @@ -155,13 +155,7 @@ func (r *API) ResolveAlias(ctx context.Context, alias string) (ResolvedAlias, er return ResolvedAlias{}, err } if !r.isLocalAlias(alias) { - if !r.federationEnabled() { - return ResolvedAlias{}, fmt.Errorf( - "%w: %s belongs to another server, and resolving a remote alias", - ErrFederationDisabled, alias) - } - return ResolvedAlias{}, fmt.Errorf( - "%w: resolving an alias of another server is not implemented yet", ErrFederationDisabled) + return r.resolveRemoteAlias(ctx, alias) } stored, err := r.store.Alias(ctx, alias) @@ -282,12 +276,7 @@ type PublicRoomsResponse struct { // the token says. func (r *API) PublicRooms(ctx context.Context, req PublicRoomsRequest) (PublicRoomsResponse, error) { if req.Server != "" && req.Server != r.serverName { - if !r.federationEnabled() { - return PublicRoomsResponse{}, fmt.Errorf( - "%w: reading the room directory of %s", ErrFederationDisabled, req.Server) - } - return PublicRoomsResponse{}, fmt.Errorf( - "%w: reading another server's room directory is not implemented yet", ErrFederationDisabled) + return r.remotePublicRooms(ctx, req) } offset, err := parseDirectoryToken(req.Since) diff --git a/server/internal/roomserver/federated.go b/server/internal/roomserver/federated.go new file mode 100644 index 0000000..078b894 --- /dev/null +++ b/server/internal/roomserver/federated.go @@ -0,0 +1,689 @@ +package roomserver + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sync" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/auth" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/matrix/stateres" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// The intake of events from other servers. +// +// An event created here is built on the room's current state and can only +// fail the authorisation rules. An event from somewhere else arrives with +// none of that settled: it was built on whatever its sender knew, its auth +// events and its prev_events may be events this server has never seen, +// and it may be perfectly valid where it was made and wrong for the room as +// this server knows it. The spec's procedure for it, which this file +// follows step by step, is: +// +// 1. It is well-formed, signed by its sender's server, and its content +// matches its hash — or it is replaced by its redacted form. +// 2. It passes the authorisation rules against its own auth_events, +// which are fetched if they are missing. +// 3. It passes them against the room's state *before* it — the state +// after its prev_events, resolved when there are several. A prev_event +// this server does not have is asked for with /get_missing_events and, +// if the gap is too wide, replaced by asking for the state at it. +// 4. It passes them against the room's *current* state. One that fails +// only here is soft-failed: kept, so that other events can cite it, but +// never shown to a client and never a forward extremity. +// +// Failing 1 to 3 is a rejection, and a rejected event is not stored. + +// Bounds on how much one event can make this server fetch. +const ( + // missingEventsLimit is how many events /get_missing_events is asked + // for. A gap wider than this is closed by fetching the state instead. + missingEventsLimit = 10 + + // stateFetchByIDLimit is how many missing state events are fetched one + // at a time with /event before it is cheaper to ask for the whole + // state with /state. + stateFetchByIDLimit = 50 + + // fetchConcurrency bounds the /event requests in flight at once. + fetchConcurrency = 8 +) + +// ReceivePDU takes one PDU from a /send transaction. It returns the event +// ID — which a transaction's response is keyed by — and the reason the +// event was refused, if it was. +func (r *API) ReceivePDU(ctx context.Context, origin string, raw json.RawMessage) (string, error) { + if err := r.requireFederation("receiving events from another server"); err != nil { + return "", err + } + roomID, err := roomIDOfRaw(raw) + if err != nil { + return "", err + } + room, rules, err := r.roomAndRules(ctx, roomID) + if errors.Is(err, ErrNoSuchRoom) { + return "", fmt.Errorf("%w: %s", ErrNotInRoom, roomID) + } else if err != nil { + return "", err + } + e, err := parseRemote(raw, rules) + if err != nil { + return "", err + } + if in, err := r.participating(ctx, roomID); err != nil { + return e.EventID(), err + } else if !in { + return e.EventID(), fmt.Errorf("%w: this server is not in %s", ErrNotInRoom, roomID) + } + if err := r.requireAllowed(ctx, roomID, origin); err != nil { + return e.EventID(), err + } + return e.EventID(), r.receive(ctx, incoming{origin: origin, room: room, rules: rules, event: e, fetchMissing: true}) +} + +// incoming is one event on its way in. +type incoming struct { + origin string + room storage.Room + rules roomversion.Rules + event *event.Event + + // fetchMissing allows /get_missing_events. It is off for the events + // that call itself returns, so that a gap is closed one level deep and + // no further. + fetchMissing bool + + // relay marks an event handed over by send_join or send_leave; see + // Notification.Relay. + relay bool + + // batch is the part of a backfill batch already accepted, which is + // not stored until the whole batch is (see backfill.go). The events in + // it are this event's likeliest auth and prev events, so they are + // looked for here before the store and before asking anybody. + batch map[string]batchEvent +} + +// receive runs the intake procedure. +func (r *API) receive(ctx context.Context, in incoming) error { + // An event already held, as an outlier or in the timeline, has been + // through this before. An outlier is left an outlier even when it + // arrives again as an ordinary event: it is one because this server + // used it as state without having its history, and making it a forward + // extremity now would put the room's head on an event whose children + // may already be here. + if _, err := r.store.Event(ctx, in.room.ID, in.event.EventID()); err == nil { + return nil + } else if !isNotFound(err) { + return err + } + + e, err := r.verifyIncoming(ctx, in.event) + if err != nil { + return err + } + in.event = e + + // Step 2: the event against its own auth events. + authEvents, err := r.authEventsOf(ctx, in) + if err != nil { + return err + } + create, err := r.createEvent(ctx, in.room, in.rules) + if err != nil { + return err + } + if err := auth.CheckAuthEvents(e, authEvents); err != nil { + return err + } + if err := auth.Check(ctx, e, withCreate(auth.NewState(authEvents), create), r.verifier()); err != nil { + return err + } + + // Step 3: the event against the state before it. + stateBefore, err := r.stateBeforeIncoming(ctx, in) + if err != nil { + return err + } + if err := auth.Check(ctx, e, withCreate(stateBefore, create), r.verifier()); err != nil { + return fmt.Errorf("%w (against the state before it)", err) + } + + return r.persistIncoming(ctx, in, stateBefore) +} + +// withCreate adds the create event to a state that lacks it, which from +// room version 12 is every state built from auth_events. +func withCreate(st auth.State, create *event.Event) auth.State { + if st.Create() == nil && create != nil { + st[auth.StateKey{Type: event.TypeCreate}] = create + } + return st +} + +// createEvent reads a room's create event. +func (r *API) createEvent(ctx context.Context, room storage.Room, rules roomversion.Rules) (*event.Event, error) { + stored, err := r.store.CurrentStateEvent(ctx, room.ID, event.TypeCreate, "") + if err != nil { + if isNotFound(err) { + return nil, fmt.Errorf("%w: this server does not hold the state of %s", ErrNotInRoom, room.ID) + } + return nil, err + } + return parseEvent(stored, rules) +} + +// persistIncoming is step 4 and the write: under the room lock, the event +// is checked against the current state and either soft-failed or added to +// the room's head. +func (r *API) persistIncoming(ctx context.Context, in incoming, stateBefore auth.State) error { + e := in.event + mu := r.lock(in.room.ID) + mu.Lock() + defer mu.Unlock() + + // Another transaction may have delivered the same event while this one + // was fetching what it needed. + if _, err := r.store.Event(ctx, in.room.ID, e.EventID()); err == nil { + return nil + } else if !isNotFound(err) { + return err + } + + current, extremities, _, err := r.roomState(ctx, in.room, in.rules) + if err != nil { + return err + } + + // Step 4: soft failure. + if err := auth.Check(ctx, e, current, r.verifier()); err != nil { + r.log.Info("soft-failed an event from another server", + "room_id", in.room.ID, "event_id", e.EventID(), "type", e.Type(), + "sender", e.Sender(), "origin", in.origin, "reason", err) + return r.store.WriteOutliers(ctx, in.room.ID, []storage.StoredEvent{storedOf(in.room.ID, e)}) + } + + // The new forward extremities are the old ones this event does not + // descend from, plus the event. + prevs := e.PrevEventIDs() + var newExtremities []string + for _, x := range extremities { + if !slices.Contains(prevs, x.EventID()) { + newExtremities = append(newExtremities, x.EventID()) + } + } + newExtremities = append(newExtremities, e.EventID()) + + req := persistRequest{ + Room: in.room, + Rules: in.rules, + Event: e, + StateBefore: stateBefore, + Extremities: newExtremities, + Origin: in.origin, + Relay: in.relay, + } + if e.Type() == event.TypeRedaction { + req.RedactsEventID = e.Redacts() + } + + // The ordinary case: the event follows the room's single extremity, so + // the state before it is the current state and the state after it is + // one tuple more. Anything else — a fork, or an event built on an + // older part of the room — means working the current state out again + // from the room's new head. + linear := len(extremities) == 1 && len(prevs) == 1 && prevs[0] == extremities[0].EventID() + if !linear { + resolved, err := r.stateAtHead(ctx, in.room, in.rules, newExtremities, e, stateBefore) + if err != nil { + return err + } + req.CurrentState = resolved + } + return r.persist(ctx, req) +} + +// stateAtHead resolves the room's state across a set of forward +// extremities, one of which is the event being persisted. +func (r *API) stateAtHead(ctx context.Context, room storage.Room, rules roomversion.Rules, extremityIDs []string, e *event.Event, stateBefore auth.State) (auth.State, error) { + var sets [][]*event.Event + for _, id := range extremityIDs { + if id == e.EventID() { + set := stateres.Sorted(stateBefore) + if e.IsState() { + set = append(set, e) + } + sets = append(sets, set) + continue + } + stored, err := r.store.Event(ctx, room.ID, id) + if err != nil { + return nil, err + } + x, err := parseEvent(stored, rules) + if err != nil { + return nil, err + } + set, err := r.stateAfter(ctx, room.ID, rules, x) + if err != nil { + return nil, err + } + sets = append(sets, set) + } + if len(sets) == 1 { + return auth.NewState(sets[0]), nil + } + resolved, err := stateres.Resolve(ctx, rules, sets, eventProvider{store: r.store, roomID: room.ID, rules: rules}) + if err != nil { + return nil, fmt.Errorf("resolving the state of %s: %w", room.ID, err) + } + return resolved, nil +} + +// authEventsOf returns an incoming event's auth events, fetching the ones +// this server does not hold. +func (r *API) authEventsOf(ctx context.Context, in incoming) ([]*event.Event, error) { + ids := in.event.AuthEventIDs() + have, err := r.heldOrBatched(ctx, in, ids) + if err != nil { + return nil, err + } + if len(have) < len(ids) { + if err := r.fetchAuthChain(ctx, in); err != nil { + return nil, err + } + if have, err = r.heldOrBatched(ctx, in, ids); err != nil { + return nil, err + } + } + if len(have) < len(ids) { + return nil, fmt.Errorf("%w: %s cites auth events that %s could not supply", + ErrBadEvent, in.event.EventID(), in.origin) + } + + // Keep the order the event cites them in; CheckAuthEvents does not + // care, but a log line naming them does. + byID := map[string]*event.Event{} + for _, e := range have { + byID[e.EventID()] = e + } + out := make([]*event.Event, 0, len(ids)) + for _, id := range ids { + out = append(out, byID[id]) + } + return out, nil +} + +// heldOrBatched is eventsByID that also finds the events of the backfill +// batch an incoming event belongs to. +func (r *API) heldOrBatched(ctx context.Context, in incoming, ids []string) ([]*event.Event, error) { + var out, rest []string + for _, id := range ids { + if _, ok := in.batch[id]; ok { + out = append(out, id) + } else { + rest = append(rest, id) + } + } + held, err := r.eventsByID(ctx, in.room.ID, in.rules, rest) + if err != nil { + return nil, err + } + for _, id := range out { + held = append(held, in.batch[id].event) + } + return held, nil +} + +// eventsByID reads and parses the events of a room this server holds, +// outliers included, leaving out the ones it does not. +func (r *API) eventsByID(ctx context.Context, roomID string, rules roomversion.Rules, ids []string) ([]*event.Event, error) { + if len(ids) == 0 { + return nil, nil + } + stored, err := r.store.EventsByID(ctx, roomID, ids) + if err != nil { + return nil, err + } + return parseEvents(stored, rules) +} + +// fetchAuthChain asks the origin for an event's whole auth chain and +// stores what passes as outliers. +func (r *API) fetchAuthChain(ctx context.Context, in incoming) error { + resp, err := r.remote.EventAuth(ctx, in.origin, in.room.ID, in.event.EventID()) + if err != nil { + return remoteError(in.origin, err) + } + return r.acceptOutliers(ctx, in.room, in.rules, resp.AuthChain) +} + +// acceptOutliers parses, verifies and authorises a batch of events from +// another server, and stores the ones that pass as outliers. +func (r *API) acceptOutliers(ctx context.Context, room storage.Room, rules roomversion.Rules, raws []json.RawMessage) error { + var fresh []*event.Event + referenced := map[string]bool{} + ids := make([]string, 0, len(raws)) + for _, raw := range raws { + e, err := parseRemote(raw, rules) + if err != nil { + r.log.Debug("ignoring an unparseable event", "room_id", room.ID, "error", err) + continue + } + if !inRoom(e, room.ID) { + continue + } + ids = append(ids, e.EventID()) + fresh = append(fresh, e) + for _, id := range e.AuthEventIDs() { + referenced[id] = true + } + } + + // What is already here does not need fetching, verifying or storing + // again; it is only needed as the auth events of what is new. + held, err := r.eventsByID(ctx, room.ID, rules, ids) + if err != nil { + return err + } + heldIDs := map[string]bool{} + for _, e := range held { + heldIDs[e.EventID()] = true + } + var toCheck []*event.Event + for _, e := range fresh { + if !heldIDs[e.EventID()] { + toCheck = append(toCheck, e) + } + } + if len(toCheck) == 0 { + return nil + } + + verified := r.verifyAll(ctx, toCheck) + + knownIDs := make([]string, 0, len(referenced)) + for id := range referenced { + knownIDs = append(knownIDs, id) + } + knownEvents, err := r.eventsByID(ctx, room.ID, rules, knownIDs) + if err != nil { + return err + } + known := map[string]*event.Event{} + for _, e := range knownEvents { + known[e.EventID()] = e + } + if create, err := r.createEvent(ctx, room, rules); err == nil { + known[create.EventID()] = create + } + + passed, rejected := r.authCheckChain(ctx, verified, known) + if len(rejected) > 0 { + r.log.Info("rejected events in a fetched auth chain or state", + "room_id", room.ID, "rejected", len(rejected)) + } + rows := make([]storage.StoredEvent, 0, len(passed)) + for _, e := range verified { + if _, ok := passed[e.EventID()]; ok { + rows = append(rows, storedOf(room.ID, e)) + } + } + return r.store.WriteOutliers(ctx, room.ID, rows) +} + +// verifyAll checks the signatures and hashes of a batch, in parallel, and +// returns the events that pass in their original order. +func (r *API) verifyAll(ctx context.Context, events []*event.Event) []*event.Event { + out := make([]*event.Event, len(events)) + sem := make(chan struct{}, fetchConcurrency) + var wg sync.WaitGroup + for i, e := range events { + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + verified, err := r.verifyIncoming(ctx, e) + if err != nil { + r.log.Debug("dropping an event that does not verify", "event_id", e.EventID(), "error", err) + return + } + out[i] = verified + }() + } + wg.Wait() + return slices.DeleteFunc(out, func(e *event.Event) bool { return e == nil }) +} + +// stateBeforeIncoming works out the room state an incoming event was built +// on: the state after each of its prev_events, resolved together. +func (r *API) stateBeforeIncoming(ctx context.Context, in incoming) (auth.State, error) { + prevs := in.event.PrevEventIDs() + if len(prevs) == 0 { + return nil, fmt.Errorf("%w: %s has no prev_events", ErrBadEvent, in.event.EventID()) + } + + // A prev_event earlier in the same backfill batch has its state after + // it in hand already. + var sets [][]*event.Event + var stored []string + for _, id := range prevs { + if b, ok := in.batch[id]; ok { + sets = append(sets, b.stateAfter) + } else { + stored = append(stored, id) + } + } + + known, missing, err := r.splitPrevs(ctx, in.room, in.rules, stored) + if err != nil { + return nil, err + } + if len(missing) > 0 && in.fetchMissing { + r.fetchMissingEvents(ctx, in) + if known, missing, err = r.splitPrevs(ctx, in.room, in.rules, stored); err != nil { + return nil, err + } + } + + for _, prev := range known { + set, err := r.stateAfter(ctx, in.room.ID, in.rules, prev) + if err != nil { + return nil, err + } + sets = append(sets, set) + } + for _, prevID := range missing { + set, err := r.stateAfterRemote(ctx, in, prevID) + if err != nil { + return nil, err + } + sets = append(sets, set) + } + + if len(sets) == 1 { + return auth.NewState(sets[0]), nil + } + resolved, err := stateres.Resolve(ctx, in.rules, sets, + eventProvider{store: r.store, roomID: in.room.ID, rules: in.rules}) + if err != nil { + return nil, fmt.Errorf("resolving the state before %s: %w", in.event.EventID(), err) + } + return resolved, nil +} + +// splitPrevs sorts prev_events into the ones in this server's timeline, +// which have a state snapshot, and the ones it lacks. An outlier counts as +// lacking: it is held, but the state at it is not. +func (r *API) splitPrevs(ctx context.Context, room storage.Room, rules roomversion.Rules, prevs []string) ([]*event.Event, []string, error) { + stored, err := r.store.EventsByID(ctx, room.ID, prevs) + if err != nil { + return nil, nil, err + } + have := map[string]storage.StoredEvent{} + for _, s := range stored { + if !s.Outlier { + have[s.EventID] = s + } + } + var known []*event.Event + var missing []string + for _, id := range prevs { + s, ok := have[id] + if !ok { + missing = append(missing, id) + continue + } + e, err := parseEvent(s, rules) + if err != nil { + return nil, nil, err + } + known = append(known, e) + } + return known, missing, nil +} + +// fetchMissingEvents asks the origin for the events between this server's +// head of the room and the incoming event, and takes each of them in, +// oldest first. Failures are logged, not returned: whatever this closes of +// the gap helps, and what it leaves open is closed by fetching state. +func (r *API) fetchMissingEvents(ctx context.Context, in incoming) { + stored, err := r.store.Extremities(ctx, in.room.ID) + if err != nil { + r.log.Warn("could not read extremities to fetch missing events", "room_id", in.room.ID, "error", err) + return + } + earliest := make([]string, 0, len(stored)) + var minDepth int64 + for i, s := range stored { + earliest = append(earliest, s.EventID) + if i == 0 || s.Depth < minDepth { + minDepth = s.Depth + } + } + + resp, err := r.remote.MissingEvents(ctx, in.origin, in.room.ID, federation.MissingEventsRequest{ + EarliestEvents: earliest, + LatestEvents: []string{in.event.EventID()}, + Limit: missingEventsLimit, + MinDepth: minDepth, + }) + if err != nil { + r.log.Info("could not fetch missing events", "room_id", in.room.ID, + "origin", in.origin, "event_id", in.event.EventID(), "error", err) + return + } + + var events []*event.Event + for _, raw := range resp.Events { + e, err := parseRemote(raw, in.rules) + if err != nil || e.RoomID() != in.room.ID || e.EventID() == in.event.EventID() { + continue + } + events = append(events, e) + } + eventsByDepth(events) + for _, e := range events { + err := r.receive(ctx, incoming{origin: in.origin, room: in.room, rules: in.rules, event: e}) + if err != nil { + r.log.Info("a fetched missing event was not accepted", + "room_id", in.room.ID, "event_id", e.EventID(), "error", err) + } + } +} + +// stateAfterRemote asks the origin for the state at a prev_event this +// server does not have, fetches whatever of it is missing, and returns the +// state after that event. +// +// /state_ids answers with the state *before* the event it is asked about, +// so the event itself is fetched too and, if it is a state event, added on +// top. Everything fetched is authorised against its own auth events and +// stored as an outlier: it is real, but this server has none of the +// history that would put it in the timeline. +func (r *API) stateAfterRemote(ctx context.Context, in incoming, prevID string) ([]*event.Event, error) { + ids, err := r.remote.StateIDs(ctx, in.origin, in.room.ID, prevID) + if err != nil { + return nil, remoteError(in.origin, err) + } + + need := append(append(slices.Clone(ids.PDUIDs), ids.AuthChainIDs...), prevID) + have, err := r.store.EventsByID(ctx, in.room.ID, need) + if err != nil { + return nil, err + } + haveIDs := map[string]bool{} + for _, s := range have { + haveIDs[s.EventID] = true + } + var missing []string + for _, id := range need { + if !haveIDs[id] && !slices.Contains(missing, id) { + missing = append(missing, id) + } + } + + if len(missing) > 0 { + var raws []json.RawMessage + if len(missing) > stateFetchByIDLimit { + state, err := r.remote.State(ctx, in.origin, in.room.ID, prevID) + if err != nil { + return nil, remoteError(in.origin, err) + } + raws = append(append(raws, state.AuthChain...), state.PDUs...) + if !slices.Contains(missing, prevID) { + missing = nil + } else { + missing = []string{prevID} + } + } + raws = append(raws, r.fetchEvents(ctx, in.origin, missing)...) + if err := r.acceptOutliers(ctx, in.room, in.rules, raws); err != nil { + return nil, err + } + } + + state, err := r.eventsByID(ctx, in.room.ID, in.rules, ids.PDUIDs) + if err != nil { + return nil, err + } + prev, err := r.eventsByID(ctx, in.room.ID, in.rules, []string{prevID}) + if err != nil { + return nil, err + } + if len(prev) == 1 && prev[0].IsState() { + state = append(state, prev[0]) + } + return state, nil +} + +// fetchEvents fetches events one by one from a server, in parallel, +// returning the ones it supplied. +func (r *API) fetchEvents(ctx context.Context, server string, ids []string) []json.RawMessage { + results := make([]json.RawMessage, len(ids)) + sem := make(chan struct{}, fetchConcurrency) + var wg sync.WaitGroup + for i, id := range ids { + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + txn, err := r.remote.Event(ctx, server, id) + if err != nil || len(txn.PDUs) == 0 { + r.log.Debug("could not fetch an event", "server", server, "event_id", id, "error", err) + return + } + results[i] = txn.PDUs[0] + }() + } + wg.Wait() + return slices.DeleteFunc(results, func(raw json.RawMessage) bool { return raw == nil }) +} diff --git a/server/internal/roomserver/federated_directory.go b/server/internal/roomserver/federated_directory.go new file mode 100644 index 0000000..2f659f6 --- /dev/null +++ b/server/internal/roomserver/federated_directory.go @@ -0,0 +1,53 @@ +package roomserver + +import ( + "context" + "encoding/json" + "fmt" + "strings" +) + +// resolveRemoteAlias asks the server an alias belongs to what it points +// at. Remote aliases are never cached: the owning server may move one at +// any time, and a stale answer would send a join to the wrong room. +func (r *API) resolveRemoteAlias(ctx context.Context, alias string) (ResolvedAlias, error) { + if err := r.requireFederation(fmt.Sprintf("%s belongs to another server, and resolving a remote alias", alias)); err != nil { + return ResolvedAlias{}, err + } + _, server, _ := strings.Cut(alias[1:], ":") + resp, err := r.remote.QueryDirectory(ctx, server, alias) + if err != nil { + err = remoteError(server, err) + if isNotFound(err) { + return ResolvedAlias{}, fmt.Errorf("%w: %s", ErrNoSuchAlias, alias) + } + return ResolvedAlias{}, err + } + if resp.RoomID == "" { + return ResolvedAlias{}, fmt.Errorf("%w: %s answered for %s without a room ID", ErrRemote, server, alias) + } + servers := resp.Servers + if len(servers) == 0 { + servers = []string{server} + } + return ResolvedAlias{RoomID: resp.RoomID, Servers: servers}, nil +} + +// remotePublicRooms reads another server's public room directory. +func (r *API) remotePublicRooms(ctx context.Context, req PublicRoomsRequest) (PublicRoomsResponse, error) { + if err := r.requireFederation("reading the room directory of " + req.Server); err != nil { + return PublicRoomsResponse{}, err + } + raw, err := r.remote.PublicRooms(ctx, req.Server, req.Limit, req.Since, req.Search) + if err != nil { + return PublicRoomsResponse{}, remoteError(req.Server, err) + } + var resp PublicRoomsResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return PublicRoomsResponse{}, fmt.Errorf("%w: %s sent an unreadable directory: %v", ErrRemote, req.Server, err) + } + if resp.Chunk == nil { + resp.Chunk = []PublicRoom{} + } + return resp, nil +} diff --git a/server/internal/roomserver/federated_membership.go b/server/internal/roomserver/federated_membership.go new file mode 100644 index 0000000..9e3bb21 --- /dev/null +++ b/server/internal/roomserver/federated_membership.go @@ -0,0 +1,787 @@ +package roomserver + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/auth" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" + "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Membership across servers. +// +// A server can only sign as itself, and a room's state lives with the +// servers in it, so every membership change that crosses a server +// boundary is a little protocol of its own: +// +// - Joining a room this server is not in: ask a resident server for a +// join template (make_join), fill it in and sign it, and hand it back +// (send_join), which returns the room's state. +// - Leaving one it is not in — rejecting an invite, in practice — is the +// same with make_leave and send_leave. +// - Inviting a user of another server: their server has to countersign +// the invite before it can go into the room. +// +// And the other side of each, for when the other server is the one asking. + +// joinNeedsFederation reports whether a join has to go through another +// server: this server does not hold the room, holds only an invite to it, +// or holds a stale copy of a room it has left that still has remote +// members to ask. +func (r *API) joinNeedsFederation(ctx context.Context, req JoinRequest) (bool, error) { + if _, err := r.store.Room(ctx, req.RoomID); errors.Is(err, ErrNoSuchRoom) { + return true, nil + } else if err != nil { + return false, err + } + if _, err := r.store.CurrentStateEvent(ctx, req.RoomID, event.TypeCreate, ""); isNotFound(err) { + return true, nil + } else if err != nil { + return false, err + } + in, err := r.participating(ctx, req.RoomID) + if err != nil || in { + return false, err + } + // Every local user has left. A room that only ever had local members + // is still this server's to rejoin on its own; one with members + // elsewhere has moved on without us, and has to be asked. + servers, err := r.JoinedServers(ctx, req.RoomID) + if err != nil { + return false, err + } + return slices.ContainsFunc(servers, func(s string) bool { return s != r.serverName }), nil +} + +// joinRemote joins a room through another server, trying each candidate +// in turn. +func (r *API) joinRemote(ctx context.Context, req JoinRequest) (string, error) { + if err := r.requireFederation(fmt.Sprintf("%s is not a room on this server, and joining a remote room", req.RoomID)); err != nil { + return "", err + } + servers := r.candidateServers(ctx, req.RoomID, req.ServerNames) + if len(servers) == 0 { + return "", fmt.Errorf("%w: no server to join %s through", ErrNoSuchRoom, req.RoomID) + } + + var firstErr error + for _, server := range servers { + err := r.joinVia(ctx, server, req) + if err == nil { + return req.RoomID, nil + } + r.log.Info("joining through a server failed", + "room_id", req.RoomID, "server", server, "user_id", req.User, "error", err) + // The first server's answer is the one worth reporting: it is the + // one the client named, or the room's own. + if firstErr == nil { + firstErr = err + } + } + return "", firstErr +} + +// eventTemplate is the part of a make_join or make_leave answer the +// joining server uses. +type eventTemplate struct { + RoomID string `json:"room_id"` + Type string `json:"type"` + StateKey *string `json:"state_key"` + Sender string `json:"sender"` + Content json.RawMessage `json:"content"` + PrevEvents []string `json:"prev_events"` + AuthEvents []string `json:"auth_events"` + Depth int64 `json:"depth"` +} + +// buildFromTemplate fills in and signs a membership template from another +// server. The template's DAG references are kept as given — they are what +// the resident server knows and this one does not — and its content is +// merged with this server's own, so that a restricted join keeps the +// authorisation the resident server chose. +func (r *API) buildFromTemplate(raw json.RawMessage, rules roomversion.Rules, roomID, userID, membership string, extra map[string]any) (*event.Event, error) { + var tmpl eventTemplate + if err := json.Unmarshal(raw, &tmpl); err != nil { + return nil, fmt.Errorf("%w: the event template is unreadable: %v", ErrRemote, err) + } + if tmpl.Type != event.TypeMember || tmpl.StateKey == nil || *tmpl.StateKey != userID || + tmpl.Sender != userID || tmpl.RoomID != roomID { + return nil, fmt.Errorf("%w: the event template is not a membership event for %s in %s", ErrRemote, userID, roomID) + } + + content := map[string]any{} + if len(tmpl.Content) > 0 { + if err := json.Unmarshal(tmpl.Content, &content); err != nil { + return nil, fmt.Errorf("%w: the template content is not an object", ErrRemote) + } + } + for k, v := range extra { + content[k] = v + } + content["membership"] = membership + + builder := event.Builder{ + RoomID: roomID, + Type: event.TypeMember, + StateKey: statePtr(userID), + Sender: userID, + Content: mustMarshal(content), + PrevEvents: tmpl.PrevEvents, + AuthEvents: tmpl.AuthEvents, + Depth: tmpl.Depth, + } + built, err := builder.Build(r.now(), r.signingKey, rules) + if err != nil { + return nil, fmt.Errorf("%w: the template does not make a valid event: %v", ErrRemote, err) + } + return built, nil +} + +// joinVia runs make_join and send_join against one server. +func (r *API) joinVia(ctx context.Context, server string, req JoinRequest) error { + tmpl, err := r.remote.MakeJoin(ctx, server, req.RoomID, req.User, supportedVersionStrings()) + if err != nil { + return remoteError(server, err) + } + version := tmpl.RoomVersion + if version == "" { + // The spec's default for a server too old to say. + version = "1" + } + rules, err := roomversion.Get(roomversion.Version(version)) + if err != nil { + return fmt.Errorf("%w: %s is version %q: %v", ErrIncompatibleRoomVersion, req.RoomID, version, err) + } + + // The profile and the reason are this server's to add; anything about + // authorisation is the template's. + extra := r.membershipContentMap(ctx, event.MembershipJoin, req.User, req.Reason, false) + delete(extra, "membership") + join, err := r.buildFromTemplate(tmpl.Event, rules, req.RoomID, req.User, event.MembershipJoin, extra) + if err != nil { + return err + } + + resp, err := r.remote.SendJoin(ctx, server, req.RoomID, join.EventID(), join.JSON()) + if err != nil { + return remoteError(server, err) + } + // A restricted join comes back countersigned by the server that + // authorised it, and that copy is the one the room will hold. + if len(resp.Event) > 0 { + countersigned, err := parseRemote(resp.Event, rules) + if err == nil && countersigned.EventID() == join.EventID() { + join = countersigned + } + } + return r.completeJoin(ctx, server, req.RoomID, rules, join, resp) +} + +// completeJoin takes in the room a send_join answer describes, and the +// join on top of it. +// +// Every event of the state and auth chain is verified and authorised +// against its own auth events before any of it is believed: the resident +// server is trusted to know the room, not to tell the truth about it. +func (r *API) completeJoin(ctx context.Context, server, roomID string, rules roomversion.Rules, join *event.Event, resp federation.SendJoinResponse) error { + byID := map[string]*event.Event{} + var all []*event.Event + stateIDs := map[string]bool{} + for i, raw := range append(slices.Clone(resp.State), resp.AuthChain...) { + e, err := parseRemote(raw, rules) + if err != nil || !inRoom(e, roomID) { + continue + } + if i < len(resp.State) { + stateIDs[e.EventID()] = true + } + if _, dup := byID[e.EventID()]; dup { + continue + } + byID[e.EventID()] = e + all = append(all, e) + } + + verified := r.verifyAll(ctx, all) + passed, rejected := r.authCheckChain(ctx, verified, nil) + if len(rejected) > 0 { + r.log.Info("rejected events in a send_join answer", + "room_id", roomID, "server", server, "rejected", len(rejected)) + } + + var stateEvents []*event.Event + for _, e := range verified { + if stateIDs[e.EventID()] && e.IsState() { + if _, ok := passed[e.EventID()]; ok { + stateEvents = append(stateEvents, e) + } + } + } + state := auth.NewState(stateEvents) + create := state.Create() + if create == nil { + return fmt.Errorf("%w: %s sent the state of %s without its create event", ErrRemote, server, roomID) + } + if err := auth.Check(ctx, join, state, r.verifier()); err != nil { + return fmt.Errorf("the room as %s describes it does not allow the join: %w", server, err) + } + + mu := r.lock(roomID) + mu.Lock() + defer mu.Unlock() + + if err := r.store.CreateRoom(ctx, storage.NewRoom{ + ID: roomID, + Version: string(rules.Version), + Creator: create.Sender(), + CreatedAtMS: r.nowMS(), + }); err != nil && !errors.Is(err, storage.ErrRoomExists) { + return err + } + room, err := r.store.Room(ctx, roomID) + if err != nil { + return err + } + if room.Version != string(rules.Version) { + return fmt.Errorf("%w: this server knows %s as version %s, and %s says %s", + ErrIncompatibleRoomVersion, roomID, room.Version, server, rules.Version) + } + + rows := make([]storage.StoredEvent, 0, len(passed)) + for _, e := range verified { + if _, ok := passed[e.EventID()]; ok { + rows = append(rows, storedOf(roomID, e)) + } + } + if err := r.store.WriteOutliers(ctx, roomID, rows); err != nil { + return err + } + + stateKey, _ := join.StateKey() + if err := r.persist(ctx, persistRequest{ + Room: room, + Rules: rules, + Event: join, + StateBefore: state, + CurrentState: withState(state, event.TypeMember, stateKey, join), + // The resident server has the join already and passes it on to + // everybody else; sending it from here as well would only double + // the traffic. + Origin: server, + }); err != nil { + return err + } + r.log.Info("joined a room over federation", + "room_id", roomID, "via", server, "user_id", join.Sender(), + "state_events", len(stateEvents), "auth_chain", len(resp.AuthChain)) + return nil +} + +// leaveRemote leaves a room this server is not in — rejects an invite, +// that is — through a server that is. +// +// If no server can be reached the invite is still rejected locally: a +// leave is built and stored here and nowhere else. The inviting server +// never hears of it, which is the lesser problem; an invite the user +// cannot get rid of because its server is gone is the greater one. +func (r *API) leaveRemote(ctx context.Context, roomID, userID, reason string) error { + room, rules, err := r.roomAndRules(ctx, roomID) + if err != nil { + return err + } + membership, err := r.store.Membership(ctx, roomID, userID) + if err != nil { + return err + } + if membership.Membership != event.MembershipInvite && membership.Membership != event.MembershipKnock { + return fmt.Errorf("%w: %s is not in %s", ErrForbidden, userID, roomID) + } + + extra := map[string]any{} + if reason != "" { + extra["reason"] = reason + } + if r.federating() { + for _, server := range r.candidateServers(ctx, roomID, nil) { + leave, err := r.leaveVia(ctx, server, rules, roomID, userID, extra) + if err != nil { + r.log.Info("leaving through a server failed", + "room_id", roomID, "server", server, "user_id", userID, "error", err) + continue + } + return r.persistOutOfBand(ctx, room, leave, nil, server) + } + } + + // Nobody could be asked: reject locally, on top of the invite. + inviteStored, err := r.store.Event(ctx, roomID, membership.EventID) + if err != nil { + return err + } + invite, err := parseEvent(inviteStored, rules) + if err != nil { + return err + } + extra["membership"] = event.MembershipLeave + builder := event.Builder{ + RoomID: roomID, + Type: event.TypeMember, + StateKey: statePtr(userID), + Sender: userID, + Content: mustMarshal(extra), + PrevEvents: []string{invite.EventID()}, + AuthEvents: []string{invite.EventID()}, + Depth: nextDepth([]*event.Event{invite}), + } + leave, err := builder.Build(r.now(), r.signingKey, rules) + if err != nil { + return err + } + r.log.Info("rejected an invite locally; no server in the room could be reached", + "room_id", roomID, "user_id", userID) + return r.persistOutOfBand(ctx, room, leave, nil, "") +} + +// leaveVia runs make_leave and send_leave against one server. +func (r *API) leaveVia(ctx context.Context, server string, rules roomversion.Rules, roomID, userID string, extra map[string]any) (*event.Event, error) { + tmpl, err := r.remote.MakeLeave(ctx, server, roomID, userID) + if err != nil { + return nil, remoteError(server, err) + } + leave, err := r.buildFromTemplate(tmpl.Event, rules, roomID, userID, event.MembershipLeave, extra) + if err != nil { + return nil, err + } + if err := r.remote.SendLeave(ctx, server, roomID, leave.EventID(), leave.JSON()); err != nil { + return nil, remoteError(server, err) + } + return leave, nil +} + +// persistOutOfBand stores a membership event for a room this server is not +// in: an invite it was asked to countersign, or the rejection of one. +// +// The event is an outlier — there is no state here to put it in the +// timeline against — and still becomes the user's membership in the +// room's current state, which is the one thing /sync needs to tell them +// they were invited, or are no longer. +func (r *API) persistOutOfBand(ctx context.Context, room storage.Room, e *event.Event, inviteState []byte, origin string) error { + mu := r.lock(room.ID) + mu.Lock() + defer mu.Unlock() + + stateKey, _ := e.StateKey() + membership, err := e.Membership() + if err != nil { + return err + } + stored := storedOf(room.ID, e) + ordering, err := r.store.Write(ctx, storage.RoomWrite{ + Event: stored, + Outlier: true, + StateDelta: []storage.StateEntry{{Type: event.TypeMember, StateKey: stateKey, EventID: e.EventID(), Membership: membership}}, + InviteState: inviteState, + }) + if err != nil { + return err + } + if r.notifier != nil { + r.notifier.EventPersisted(ctx, Notification{ + RoomID: room.ID, + Event: e, + StreamOrdering: ordering, + StateChanged: true, + Origin: origin, + }) + } + return nil +} + +// inviteRemote sends an invite to a user of another server through the +// pipeline, with their server's countersignature taken under the room +// lock. +func (r *API) inviteRemote(ctx context.Context, req InviteRequest) error { + if err := r.requireFederation(fmt.Sprintf("%s is on another server, and inviting a remote user", req.Target)); err != nil { + return err + } + targetServer, err := id.ServerNameOf(req.Target) + if err != nil { + return fmt.Errorf("%w: %v", ErrInvalidContent, err) + } + room, rules, err := r.roomAndRules(ctx, req.RoomID) + if err != nil { + return err + } + + _, err = r.send(ctx, sendRequest{ + RoomID: req.RoomID, + Type: event.TypeMember, + StateKey: statePtr(req.Target), + Sender: req.Sender, + Content: r.membershipContent(ctx, event.MembershipInvite, req.Target, req.Reason, req.IsDirect), + countersign: func(ctx context.Context, invite *event.Event) (*event.Event, error) { + stripped, err := r.strippedState(ctx, room.ID, rules, []string{req.Sender}) + if err != nil { + return nil, err + } + strippedRaw := make([]json.RawMessage, 0, len(stripped)) + for _, s := range stripped { + strippedRaw = append(strippedRaw, mustMarshal(s)) + } + resp, err := r.remote.Invite(ctx, targetServer, room.ID, invite.EventID(), federation.InviteRequest{ + RoomVersion: string(rules.Version), + Event: invite.JSON(), + InviteRoomState: strippedRaw, + }) + if err != nil { + return nil, remoteError(targetServer, err) + } + signed, err := parseRemote(resp.Event, rules) + if err != nil { + return nil, fmt.Errorf("%w: %s countersigned something unreadable: %v", ErrRemote, targetServer, err) + } + if signed.EventID() != invite.EventID() { + return nil, fmt.Errorf("%w: %s returned a different event from the invite", ErrRemote, targetServer) + } + if err := r.verifier().VerifyEventSignature(ctx, targetServer, signed); err != nil { + return nil, fmt.Errorf("%w: %s did not countersign the invite: %v", ErrRemote, targetServer, err) + } + return signed, nil + }, + }) + return err +} + +// --------------------------------------------------------------------------- +// The other side: requests from servers joining, leaving and inviting. + +// MakeJoin answers GET /make_join: a join template for a user of origin. +func (r *API) MakeJoin(ctx context.Context, origin, roomID, userID string, versions []string) (string, json.RawMessage, error) { + return r.makeMembership(ctx, origin, roomID, userID, event.MembershipJoin, versions) +} + +// MakeLeave answers GET /make_leave. +func (r *API) MakeLeave(ctx context.Context, origin, roomID, userID string) (string, json.RawMessage, error) { + return r.makeMembership(ctx, origin, roomID, userID, event.MembershipLeave, nil) +} + +// makeMembership builds a membership template for another server's user. +// +// The template is the event as this server would build it, minus the +// signature and hashes only the user's own server can add. It is checked +// against the room before it is handed out, so that a banned user is told +// so at make_join rather than after signing and sending a join that was +// never going to be accepted. +func (r *API) makeMembership(ctx context.Context, origin, roomID, userID, membership string, versions []string) (string, json.RawMessage, error) { + if err := r.requireFederation("answering another server's membership request"); err != nil { + return "", nil, err + } + if server, err := id.ServerNameOf(userID); err != nil || server != origin { + return "", nil, fmt.Errorf("%w: %s cannot ask on behalf of %s", ErrForbidden, origin, userID) + } + room, rules, err := r.roomAndRules(ctx, roomID) + if err != nil { + return "", nil, err + } + if in, err := r.participating(ctx, roomID); err != nil { + return "", nil, err + } else if !in { + return "", nil, fmt.Errorf("%w: this server is not in %s", ErrNotInRoom, roomID) + } + if err := r.requireAllowed(ctx, roomID, origin); err != nil { + return "", nil, err + } + if membership == event.MembershipJoin { + if len(versions) == 0 { + versions = []string{"1"} + } + if !slices.Contains(versions, room.Version) { + return "", nil, fmt.Errorf("%w: %s is version %s", ErrIncompatibleRoomVersion, roomID, room.Version) + } + } + + mu := r.lock(roomID) + mu.Lock() + defer mu.Unlock() + + st, extremities, _, err := r.roomState(ctx, room, rules) + if err != nil { + return "", nil, err + } + content := map[string]any{"membership": membership} + if membership == event.MembershipJoin { + authoriser, err := r.restrictedJoinAuthoriser(ctx, JoinRequest{RoomID: roomID, User: userID}) + if err != nil { + return "", nil, err + } + if authoriser != "" { + content["join_authorised_via_users_server"] = authoriser + } + } + contentRaw := mustMarshal(content) + + authEvents, err := auth.SelectAuthEvents(rules, event.TypeMember, userID, &userID, contentRaw, st) + if err != nil { + return "", nil, fmt.Errorf("%w: %s", ErrRejected, err) + } + + tmpl := map[string]any{ + "room_id": roomID, + "type": event.TypeMember, + "state_key": userID, + "sender": userID, + "content": contentRaw, + "prev_events": eventIDs(extremities), + "auth_events": eventIDs(authEvents), + "depth": nextDepth(extremities), + "origin_server_ts": r.nowMS(), + "origin": r.serverName, + } + + // A trial build, signed with this server's key in the user's name. The + // signature is worthless and nothing here checks it: what is being + // asked is only whether the room would let this user in or out. + trial, err := (&event.Builder{ + RoomID: roomID, Type: event.TypeMember, StateKey: &userID, Sender: userID, + Content: contentRaw, PrevEvents: eventIDs(extremities), AuthEvents: eventIDs(authEvents), + Depth: nextDepth(extremities), + }).Build(r.now(), r.signingKey, rules) + if err != nil { + return "", nil, err + } + if _, ok := content["join_authorised_via_users_server"]; ok { + // The real join will carry this server's countersignature, added + // at send_join; the trial needs it too, or the rule that checks + // for it refuses the join this server is about to authorise. + if trial, err = trial.Sign(r.serverName, r.signingKey); err != nil { + return "", nil, err + } + } + if err := auth.Check(ctx, trial, checkState(authEvents, st), r.verifier()); err != nil { + return "", nil, err + } + + return room.Version, mustMarshal(tmpl), nil +} + +// SendJoinResult is what PUT /send_join answers with. +type SendJoinResult struct { + State []json.RawMessage + AuthChain []json.RawMessage + + // Event is the join countersigned by this server, for a restricted + // join it authorised; nil otherwise. + Event json.RawMessage + + ServersInRoom []string +} + +// SendJoin takes a signed join from another server's user into the room +// and returns the room as of the join. +func (r *API) SendJoin(ctx context.Context, origin, roomID, eventID string, raw json.RawMessage) (SendJoinResult, error) { + e, room, rules, err := r.receiveMembership(ctx, origin, roomID, eventID, raw, event.MembershipJoin) + if err != nil { + return SendJoinResult{}, err + } + + var countersigned json.RawMessage + var content struct { + AuthorisedVia string `json:"join_authorised_via_users_server"` + } + _ = json.Unmarshal(e.Content(), &content) + if content.AuthorisedVia != "" && r.isLocal(content.AuthorisedVia) { + if e, err = e.Sign(r.serverName, r.signingKey); err != nil { + return SendJoinResult{}, err + } + countersigned = e.JSON() + } + + if err := r.receive(ctx, incoming{origin: origin, room: room, rules: rules, event: e, relay: true}); err != nil { + return SendJoinResult{}, err + } + + stored, err := r.store.Event(ctx, roomID, e.EventID()) + if err != nil { + return SendJoinResult{}, err + } + if stored.Outlier { + return SendJoinResult{}, fmt.Errorf("%w: the join was not accepted by the room's current state", ErrForbidden) + } + stateIDs, err := r.stateIDsBefore(ctx, roomID, e.EventID()) + if err != nil { + return SendJoinResult{}, err + } + state, err := r.rawEvents(ctx, roomID, stateIDs) + if err != nil { + return SendJoinResult{}, err + } + chainIDs, err := r.authChainIDs(ctx, roomID, rules, append(slices.Clone(stateIDs), e.AuthEventIDs()...)) + if err != nil { + return SendJoinResult{}, err + } + chain, err := r.rawEvents(ctx, roomID, chainIDs) + if err != nil { + return SendJoinResult{}, err + } + servers, err := r.JoinedServers(ctx, roomID) + if err != nil { + return SendJoinResult{}, err + } + return SendJoinResult{State: state, AuthChain: chain, Event: countersigned, ServersInRoom: servers}, nil +} + +// SendLeave takes a signed leave from another server's user into the +// room. +func (r *API) SendLeave(ctx context.Context, origin, roomID, eventID string, raw json.RawMessage) error { + e, room, rules, err := r.receiveMembership(ctx, origin, roomID, eventID, raw, event.MembershipLeave) + if err != nil { + return err + } + return r.receive(ctx, incoming{origin: origin, room: room, rules: rules, event: e, relay: true}) +} + +// receiveMembership checks a send_join or send_leave body: the event the +// path names, a membership event of the right kind, by a user of the +// server sending it, about themselves. +func (r *API) receiveMembership(ctx context.Context, origin, roomID, eventID string, raw json.RawMessage, membership string) (*event.Event, storage.Room, roomversion.Rules, error) { + fail := func(err error) (*event.Event, storage.Room, roomversion.Rules, error) { + return nil, storage.Room{}, roomversion.Rules{}, err + } + if err := r.requireFederation("answering another server's membership request"); err != nil { + return fail(err) + } + room, rules, err := r.roomAndRules(ctx, roomID) + if err != nil { + return fail(err) + } + if in, err := r.participating(ctx, roomID); err != nil { + return fail(err) + } else if !in { + return fail(fmt.Errorf("%w: this server is not in %s", ErrNotInRoom, roomID)) + } + if err := r.requireAllowed(ctx, roomID, origin); err != nil { + return fail(err) + } + e, err := parseRemote(raw, rules) + if err != nil { + return fail(err) + } + stateKey, _ := e.StateKey() + got, _ := e.Membership() + sender, _ := e.SenderServer() + switch { + case e.EventID() != eventID: + return fail(fmt.Errorf("%w: the event is %s, not %s", ErrBadEvent, e.EventID(), eventID)) + case e.RoomID() != roomID: + return fail(fmt.Errorf("%w: the event is in %s, not %s", ErrBadEvent, e.RoomID(), roomID)) + case e.Type() != event.TypeMember || got != membership: + return fail(fmt.Errorf("%w: not a %s event", ErrBadEvent, membership)) + case stateKey != e.Sender(): + return fail(fmt.Errorf("%w: a %s must be the sender's own", ErrBadEvent, membership)) + case sender != origin: + return fail(fmt.Errorf("%w: %s cannot send a %s for %s", ErrForbidden, origin, membership, e.Sender())) + } + return e, room, rules, nil +} + +// ReceiveInvite answers PUT /invite: another server asks this one to +// countersign an invite to one of its users. +func (r *API) ReceiveInvite(ctx context.Context, origin, roomVersion string, raw json.RawMessage, inviteRoomState []json.RawMessage) (json.RawMessage, error) { + if err := r.requireFederation("receiving an invite from another server"); err != nil { + return nil, err + } + rules, err := roomversion.Get(roomversion.Version(roomVersion)) + if err != nil { + return nil, fmt.Errorf("%w: version %q: %v", ErrIncompatibleRoomVersion, roomVersion, err) + } + e, err := parseRemote(raw, rules) + if err != nil { + return nil, err + } + target, _ := e.StateKey() + membership, _ := e.Membership() + sender, _ := e.SenderServer() + switch { + case e.Type() != event.TypeMember || membership != event.MembershipInvite: + return nil, fmt.Errorf("%w: not an invite", ErrBadEvent) + case sender != origin: + return nil, fmt.Errorf("%w: %s cannot send an invite from %s", ErrForbidden, origin, e.Sender()) + case !r.isLocal(target): + return nil, fmt.Errorf("%w: %s is not a user of this server", ErrForbidden, target) + } + if localpart, _ := r.localpartOf(target); r.profiles != nil { + if _, err := r.profiles.Account(ctx, localpart); errors.Is(err, storage.ErrNoSuchUser) { + return nil, fmt.Errorf("%w: %s does not exist", ErrNoSuchEvent, target) + } else if err != nil { + return nil, err + } + } + + verified, err := r.verifyIncoming(ctx, e) + if err != nil { + return nil, err + } + signed, err := verified.Sign(r.serverName, r.signingKey) + if err != nil { + return nil, err + } + roomID := signed.RoomID() + + // A room this server is in hears about the invite like any other + // event, and judges it the same way. One it is not in gets the invite + // as an out-of-band membership with the state the inviter sent along. + in, err := r.participating(ctx, roomID) + if err != nil { + return nil, err + } + if in { + if err := r.requireAllowed(ctx, roomID, origin); err != nil { + return nil, err + } + room, rules, err := r.roomAndRules(ctx, roomID) + if err != nil { + return nil, err + } + if err := r.receive(ctx, incoming{origin: origin, room: room, rules: rules, event: signed, fetchMissing: true}); err != nil { + return nil, err + } + return signed.JSON(), nil + } + + if err := r.store.CreateRoom(ctx, storage.NewRoom{ + ID: roomID, Version: roomVersion, Creator: signed.Sender(), CreatedAtMS: r.nowMS(), + }); err != nil && !errors.Is(err, storage.ErrRoomExists) { + return nil, err + } + room, err := r.store.Room(ctx, roomID) + if err != nil { + return nil, err + } + var stripped []json.RawMessage + for _, s := range inviteRoomState { + var probe map[string]json.RawMessage + if json.Unmarshal(s, &probe) == nil && probe != nil { + stripped = append(stripped, s) + } + } + if err := r.persistOutOfBand(ctx, room, signed, mustMarshal(orEmptyRaw(stripped)), origin); err != nil { + return nil, err + } + r.log.Info("received an invite over federation", + "room_id", roomID, "user_id", target, "sender", signed.Sender()) + return signed.JSON(), nil +} + +func orEmptyRaw(s []json.RawMessage) []json.RawMessage { + if s == nil { + return []json.RawMessage{} + } + return s +} diff --git a/server/internal/roomserver/federated_query.go b/server/internal/roomserver/federated_query.go new file mode 100644 index 0000000..200b9fa --- /dev/null +++ b/server/internal/roomserver/federated_query.go @@ -0,0 +1,373 @@ +package roomserver + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "sort" + + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// The read side of the federation API: other servers asking about a room +// this one is in. +// +// Every read is answered only for a server that is in the room itself, or +// for a room anybody may read. A server outside the room asking for its +// events is asking for history its users were never shown, and the answer +// is the same 403 whether or not the event exists. + +// Bounds on the federation reads. +const ( + maxBackfillLimit = 100 + maxMissingEventsLimit = 20 +) + +// requireServerInRoom refuses a server that may not read a room. +func (r *API) requireServerInRoom(ctx context.Context, roomID, origin string) (storage.Room, roomversion.Rules, error) { + if err := r.requireFederation("answering another server"); err != nil { + return storage.Room{}, roomversion.Rules{}, err + } + room, rules, err := r.roomAndRules(ctx, roomID) + if err != nil { + return storage.Room{}, roomversion.Rules{}, err + } + if err := r.requireAllowed(ctx, roomID, origin); err != nil { + return storage.Room{}, roomversion.Rules{}, err + } + in, err := r.ServerInRoom(ctx, roomID, origin) + if err != nil { + return storage.Room{}, roomversion.Rules{}, err + } + if in { + return room, rules, nil + } + // A world-readable room may be read by anybody, servers included. + if stored, err := r.store.CurrentStateEvent(ctx, roomID, event.TypeHistoryVisibility, ""); err == nil { + var content struct { + HistoryVisibility string `json:"history_visibility"` + } + if json.Unmarshal(contentOf(stored), &content) == nil && content.HistoryVisibility == HistoryVisibilityWorldReadable { + return room, rules, nil + } + } + return storage.Room{}, roomversion.Rules{}, fmt.Errorf("%w: %s is not in %s", ErrForbidden, origin, roomID) +} + +// contentOf reads the content of a stored event without parsing it as a +// PDU, for the one property a visibility check needs. +func contentOf(stored storage.StoredEvent) json.RawMessage { + var probe struct { + Content json.RawMessage `json:"content"` + } + _ = json.Unmarshal(stored.JSON, &probe) + return probe.Content +} + +// EventForServer answers GET /event: one event, as JSON. +func (r *API) EventForServer(ctx context.Context, origin, eventID string) (json.RawMessage, error) { + stored, err := r.store.EventByID(ctx, eventID) + if err != nil { + return nil, err + } + if _, _, err := r.requireServerInRoom(ctx, stored.RoomID, origin); err != nil { + return nil, err + } + return stored.JSON, nil +} + +// StateIDsForServer answers GET /state_ids: the state before an event and +// its auth chain, as event IDs. +func (r *API) StateIDsForServer(ctx context.Context, origin, roomID, eventID string) ([]string, []string, error) { + _, rules, err := r.requireServerInRoom(ctx, roomID, origin) + if err != nil { + return nil, nil, err + } + stateIDs, err := r.stateIDsBefore(ctx, roomID, eventID) + if err != nil { + return nil, nil, err + } + chain, err := r.authChainIDs(ctx, roomID, rules, stateIDs) + if err != nil { + return nil, nil, err + } + return stateIDs, chain, nil +} + +// StateForServer answers GET /state: StateIDsForServer with the events. +func (r *API) StateForServer(ctx context.Context, origin, roomID, eventID string) ([]json.RawMessage, []json.RawMessage, error) { + stateIDs, chainIDs, err := r.StateIDsForServer(ctx, origin, roomID, eventID) + if err != nil { + return nil, nil, err + } + state, err := r.rawEvents(ctx, roomID, stateIDs) + if err != nil { + return nil, nil, err + } + chain, err := r.rawEvents(ctx, roomID, chainIDs) + if err != nil { + return nil, nil, err + } + return state, chain, nil +} + +// EventAuthForServer answers GET /event_auth: an event's auth chain. +func (r *API) EventAuthForServer(ctx context.Context, origin, roomID, eventID string) ([]json.RawMessage, error) { + _, rules, err := r.requireServerInRoom(ctx, roomID, origin) + if err != nil { + return nil, err + } + stored, err := r.store.Event(ctx, roomID, eventID) + if err != nil { + return nil, err + } + e, err := parseEvent(stored, rules) + if err != nil { + return nil, err + } + ids, err := r.authChainIDs(ctx, roomID, rules, e.AuthEventIDs()) + if err != nil { + return nil, err + } + return r.rawEvents(ctx, roomID, ids) +} + +// stateIDsBefore returns the event IDs of the state before an event — the +// snapshot it was authorised against. +func (r *API) stateIDsBefore(ctx context.Context, roomID, eventID string) ([]string, error) { + stored, err := r.store.Event(ctx, roomID, eventID) + if err != nil { + return nil, err + } + if stored.Outlier { + // An outlier has no snapshot; the honest answer is that this + // server does not know the state at it. + return nil, fmt.Errorf("%w: the state at %s is not known here", ErrNoSuchEvent, eventID) + } + entries, err := r.store.StateBefore(ctx, roomID, eventID) + if err != nil { + return nil, err + } + return stateEntryIDs(entries), nil +} + +// authChainIDs walks auth_events from a set of events to the create event +// and returns every event on the way, the starting events excluded unless +// one of the others cites them. +func (r *API) authChainIDs(ctx context.Context, roomID string, rules roomversion.Rules, from []string) ([]string, error) { + seen := map[string]bool{} + frontier := map[string]bool{} + start := map[string]*event.Event{} + + starting, err := r.eventsByID(ctx, roomID, rules, from) + if err != nil { + return nil, err + } + for _, e := range starting { + start[e.EventID()] = e + for _, id := range e.AuthEventIDs() { + frontier[id] = true + } + } + for _, id := range from { + if _, ok := start[id]; !ok { + // A starting ID that is not an event here is somebody's auth + // event reference rather than an event of the chain's own. + frontier[id] = true + } + } + + for len(frontier) > 0 { + ids := make([]string, 0, len(frontier)) + for id := range frontier { + if !seen[id] { + seen[id] = true + ids = append(ids, id) + } + } + frontier = map[string]bool{} + if len(ids) == 0 { + break + } + events, err := r.eventsByID(ctx, roomID, rules, ids) + if err != nil { + return nil, err + } + for _, e := range events { + for _, id := range e.AuthEventIDs() { + if !seen[id] { + frontier[id] = true + } + } + } + } + + // Only the IDs that are events here: a reference to something this + // server never had is not part of the answer. + all := make([]string, 0, len(seen)) + for id := range seen { + all = append(all, id) + } + present, err := r.store.EventsByID(ctx, roomID, all) + if err != nil { + return nil, err + } + out := make([]string, 0, len(present)) + for _, s := range present { + out = append(out, s.EventID) + } + sort.Strings(out) + return out, nil +} + +// rawEvents reads events as the JSON they are held as. +func (r *API) rawEvents(ctx context.Context, roomID string, ids []string) ([]json.RawMessage, error) { + if len(ids) == 0 { + return []json.RawMessage{}, nil + } + stored, err := r.store.EventsByID(ctx, roomID, ids) + if err != nil { + return nil, err + } + out := make([]json.RawMessage, 0, len(stored)) + for _, s := range stored { + out = append(out, s.JSON) + } + return out, nil +} + +// BackfillForServer answers GET /backfill: up to limit events reachable +// backwards through prev_events from the given ones, them included, +// newest first. +func (r *API) BackfillForServer(ctx context.Context, origin, roomID string, from []string, limit int) ([]json.RawMessage, error) { + _, rules, err := r.requireServerInRoom(ctx, roomID, origin) + if err != nil { + return nil, err + } + if limit <= 0 || limit > maxBackfillLimit { + limit = maxBackfillLimit + } + events, err := r.walkBack(ctx, roomID, rules, from, nil, 0, limit) + if err != nil { + return nil, err + } + sort.SliceStable(events, func(i, j int) bool { return events[i].Depth() > events[j].Depth() }) + return jsonOf(events), nil +} + +// MissingEventsForServer answers POST /get_missing_events: the events +// between earliest and latest, oldest first. +func (r *API) MissingEventsForServer(ctx context.Context, origin, roomID string, earliest, latest []string, limit int, minDepth int64) ([]json.RawMessage, error) { + _, rules, err := r.requireServerInRoom(ctx, roomID, origin) + if err != nil { + return nil, err + } + if limit <= 0 || limit > maxMissingEventsLimit { + limit = maxMissingEventsLimit + } + starting, err := r.eventsByID(ctx, roomID, rules, latest) + if err != nil { + return nil, err + } + var from []string + for _, e := range starting { + from = append(from, e.PrevEventIDs()...) + } + events, err := r.walkBack(ctx, roomID, rules, from, earliest, minDepth, limit) + if err != nil { + return nil, err + } + eventsByDepth(events) + return jsonOf(events), nil +} + +// walkBack walks prev_events breadth first from a set of events, stopping +// at the ones in stop, below minDepth, or after limit events. +// +// Outliers are skipped. Their prev_events were never fetched, and handing +// one out would invite the asker to come back for history this server +// does not have. +func (r *API) walkBack(ctx context.Context, roomID string, rules roomversion.Rules, from, stop []string, minDepth int64, limit int) ([]*event.Event, error) { + seen := map[string]bool{} + for _, id := range stop { + seen[id] = true + } + var out []*event.Event + frontier := slices.Clone(from) + for len(frontier) > 0 && len(out) < limit { + var ids []string + for _, id := range frontier { + if !seen[id] { + seen[id] = true + ids = append(ids, id) + } + } + frontier = nil + if len(ids) == 0 { + break + } + stored, err := r.store.EventsByID(ctx, roomID, ids) + if err != nil { + return nil, err + } + var level []*event.Event + for _, s := range stored { + if s.Outlier { + continue + } + e, err := parseEvent(s, rules) + if err != nil { + return nil, err + } + if e.Depth() < minDepth { + continue + } + level = append(level, e) + } + // Newest first within a level, so that the limit keeps the events + // nearest to where the walk started. + sort.SliceStable(level, func(i, j int) bool { return level[i].Depth() > level[j].Depth() }) + for _, e := range level { + if len(out) >= limit { + break + } + out = append(out, e) + frontier = append(frontier, e.PrevEventIDs()...) + } + } + return out, nil +} + +func jsonOf(events []*event.Event) []json.RawMessage { + out := make([]json.RawMessage, 0, len(events)) + for _, e := range events { + out = append(out, e.JSON()) + } + return out +} + +// QueryAliasForServer answers GET /query/directory for an alias of this +// server. +func (r *API) QueryAliasForServer(ctx context.Context, alias string) (ResolvedAlias, error) { + if err := r.requireFederation("answering another server"); err != nil { + return ResolvedAlias{}, err + } + if !r.isLocalAlias(alias) { + return ResolvedAlias{}, fmt.Errorf("%w: %s is not an alias of this server", ErrNoSuchAlias, alias) + } + stored, err := r.store.Alias(ctx, alias) + if err != nil { + return ResolvedAlias{}, err + } + servers, err := r.JoinedServers(ctx, stored.RoomID) + if err != nil { + return ResolvedAlias{}, err + } + if !slices.Contains(servers, r.serverName) { + servers = append([]string{r.serverName}, servers...) + } + return ResolvedAlias{RoomID: stored.RoomID, Servers: servers}, nil +} diff --git a/server/internal/roomserver/federated_test.go b/server/internal/roomserver/federated_test.go new file mode 100644 index 0000000..df39753 --- /dev/null +++ b/server/internal/roomserver/federated_test.go @@ -0,0 +1,1054 @@ +package roomserver_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "path/filepath" + "slices" + "sync" + "testing" + "time" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" + "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// The federation tests run several roomservers in one process, each on its +// own database, connected by a transport that calls the other server's +// federation methods directly. Nothing about HTTP or X-Matrix is under +// test here — the federation package has its own tests for that — only +// whether two roomservers that can reach each other end up agreeing about +// a room. + +// mesh is a set of servers that can reach each other. +type mesh struct { + t *testing.T + servers map[string]*peer + + // disabled turns federation off everywhere, for the switch tests. + mu sync.Mutex + disabled bool +} + +// peer is one server of the mesh. +type peer struct { + name string + api *roomserver.API + store storage.RoomStore + key *signing.KeyPair + mesh *mesh + + mu sync.Mutex + // outbox is every event this server persisted that it would send: + // its own, and the ones it relays. + outbox []roomserver.Notification +} + +func newMesh(t *testing.T, names ...string) *mesh { + t.Helper() + m := &mesh{t: t, servers: map[string]*peer{}} + for _, name := range names { + key, err := signing.Generate("k_" + name[:1]) + if err != nil { + t.Fatal(err) + } + m.servers[name] = &peer{name: name, key: key, mesh: m} + } + for _, name := range names { + m.servers[name].start(t) + } + return m +} + +func (p *peer) start(t *testing.T) { + t.Helper() + path := filepath.ToSlash(filepath.Join(t.TempDir(), p.name+".db")) + db, err := storage.Open(t.Context(), storage.Config{ + Driver: storage.DriverSQLite, + ConnectionString: "file:" + path + "?_pragma=busy_timeout(5000)", + }, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatalf("open storage: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if err := db.Migrate(t.Context()); err != nil { + t.Fatalf("migrate: %v", err) + } + + api, err := roomserver.New(roomserver.Options{ + ServerName: p.name, + Store: db.Rooms(), + SigningKey: p.key, + Profiles: anyProfile{}, + FederationEnabled: func() bool { return !p.mesh.isDisabled() }, + Notifier: p, + Remote: transport{from: p}, + Keys: keys{m: p.mesh}, + Logger: slog.New(slog.DiscardHandler), + }) + if err != nil { + t.Fatalf("roomserver.New: %v", err) + } + p.api, p.store = api, db.Rooms() +} + +func (m *mesh) isDisabled() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.disabled +} + +// anyProfile gives every local user a display name. +type anyProfile struct{} + +func (anyProfile) Account(_ context.Context, localpart string) (storage.Account, error) { + return storage.Account{Localpart: localpart, DisplayName: "User " + localpart}, nil +} + +// EventPersisted implements roomserver.Notifier: it fills the outbox the +// way the real outbound queue picks its events. +func (p *peer) EventPersisted(_ context.Context, n roomserver.Notification) { + if n.Origin != "" && !n.Relay { + return + } + p.mu.Lock() + defer p.mu.Unlock() + p.outbox = append(p.outbox, n) +} + +// takeOutbox empties the outbox. +func (p *peer) takeOutbox() []roomserver.Notification { + p.mu.Lock() + defer p.mu.Unlock() + out := p.outbox + p.outbox = nil + return out +} + +// flush delivers every outbox to the servers in each event's room, the +// way the queue would, until nothing is left to send. It returns the +// errors the receivers reported, keyed by event ID. +func (m *mesh) flush() map[string]error { + m.t.Helper() + failures := map[string]error{} + for range 20 { + sent := false + for _, name := range slices.Sorted(mapsKeys(m.servers)) { + p := m.servers[name] + for _, n := range p.takeOutbox() { + sent = true + m.deliver(p, n, failures) + } + } + if !sent { + return failures + } + } + m.t.Fatal("flush did not settle") + return nil +} + +// deliver sends one event from a server to the others in its room. +func (m *mesh) deliver(from *peer, n roomserver.Notification, failures map[string]error) { + m.t.Helper() + servers, err := from.api.JoinedServers(m.t.Context(), n.RoomID) + if err != nil { + m.t.Fatalf("JoinedServers: %v", err) + } + // A membership event also goes to the server of the user it is + // about, so that a kicked or banned server hears of it. + if key, ok := n.Event.StateKey(); ok && n.Event.Type() == event.TypeMember { + if server, err := id.ServerNameOf(key); err == nil && !slices.Contains(servers, server) { + servers = append(servers, server) + } + } + for _, server := range servers { + if server == from.name || server == n.Origin { + continue + } + to, ok := m.servers[server] + if !ok { + continue + } + if _, err := to.api.ReceivePDU(m.t.Context(), from.name, n.Event.JSON()); err != nil { + failures[n.Event.EventID()] = err + } + } +} + +// deliverOnly sends one specific event, bypassing the outbox: the tests +// that leave gaps use it. +func (m *mesh) deliverOnly(from, to string, e *event.Event) error { + _, err := m.servers[to].api.ReceivePDU(m.t.Context(), from, e.JSON()) + return err +} + +func mapsKeys[V any](m map[string]V) func(func(string) bool) { + return func(yield func(string) bool) { + for k := range m { + if !yield(k) { + return + } + } + } +} + +// keys verifies signatures with the mesh's keys, standing in for the key +// ring. +type keys struct{ m *mesh } + +func (k keys) VerifyEventSignature(_ context.Context, serverName string, e *event.Event) error { + p, ok := k.m.servers[serverName] + if !ok { + return fmt.Errorf("no key for %s", serverName) + } + return e.VerifySignature(serverName, p.key.KeyID(), p.key.Public) +} + +// transport is RemoteServers for one peer: every call goes straight to +// the destination's federation-side method, with this peer as the origin. +type transport struct{ from *peer } + +func (t transport) to(destination string) (*roomserver.API, error) { + if t.from.mesh.isDisabled() { + return nil, federation.ErrDisabled + } + p, ok := t.from.mesh.servers[destination] + if !ok { + return nil, fmt.Errorf("%s is unreachable", destination) + } + return p.api, nil +} + +// httpError turns a roomserver error into what the HTTP layer would have +// sent back, so that the caller's error mapping is exercised. +func httpError(err error) error { + switch { + case err == nil: + return nil + case errors.Is(err, roomserver.ErrIncompatibleRoomVersion): + return &federation.HTTPError{Status: http.StatusBadRequest, Code: "M_INCOMPATIBLE_ROOM_VERSION", Message: err.Error()} + case errors.Is(err, roomserver.ErrForbidden), errors.Is(err, roomserver.ErrRejected), errors.Is(err, roomserver.ErrNotInRoom): + return &federation.HTTPError{Status: http.StatusForbidden, Code: "M_FORBIDDEN", Message: err.Error()} + case errors.Is(err, roomserver.ErrNoSuchRoom), errors.Is(err, roomserver.ErrNoSuchEvent), errors.Is(err, roomserver.ErrNoSuchAlias): + return &federation.HTTPError{Status: http.StatusNotFound, Code: "M_NOT_FOUND", Message: err.Error()} + } + return &federation.HTTPError{Status: http.StatusInternalServerError, Code: "M_UNKNOWN", Message: err.Error()} +} + +func (t transport) MakeJoin(ctx context.Context, dest, roomID, userID string, versions []string) (federation.MakeJoinResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.MakeJoinResponse{}, err + } + version, tmpl, err := api.MakeJoin(ctx, t.from.name, roomID, userID, versions) + return federation.MakeJoinResponse{RoomVersion: version, Event: tmpl}, httpError(err) +} + +func (t transport) SendJoin(ctx context.Context, dest, roomID, eventID string, raw json.RawMessage) (federation.SendJoinResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.SendJoinResponse{}, err + } + res, err := api.SendJoin(ctx, t.from.name, roomID, eventID, raw) + return federation.SendJoinResponse{Origin: dest, State: res.State, AuthChain: res.AuthChain, Event: res.Event}, httpError(err) +} + +func (t transport) MakeLeave(ctx context.Context, dest, roomID, userID string) (federation.MakeJoinResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.MakeJoinResponse{}, err + } + version, tmpl, err := api.MakeLeave(ctx, t.from.name, roomID, userID) + return federation.MakeJoinResponse{RoomVersion: version, Event: tmpl}, httpError(err) +} + +func (t transport) SendLeave(ctx context.Context, dest, roomID, eventID string, raw json.RawMessage) error { + api, err := t.to(dest) + if err != nil { + return err + } + return httpError(api.SendLeave(ctx, t.from.name, roomID, eventID, raw)) +} + +func (t transport) Invite(ctx context.Context, dest, _, _ string, req federation.InviteRequest) (federation.InviteResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.InviteResponse{}, err + } + signed, err := api.ReceiveInvite(ctx, t.from.name, req.RoomVersion, req.Event, req.InviteRoomState) + return federation.InviteResponse{Event: signed}, httpError(err) +} + +func (t transport) Event(ctx context.Context, dest, eventID string) (federation.Transaction, error) { + api, err := t.to(dest) + if err != nil { + return federation.Transaction{}, err + } + raw, err := api.EventForServer(ctx, t.from.name, eventID) + if err != nil { + return federation.Transaction{}, httpError(err) + } + return federation.Transaction{Origin: dest, PDUs: []json.RawMessage{raw}}, nil +} + +func (t transport) EventAuth(ctx context.Context, dest, roomID, eventID string) (federation.EventAuthResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.EventAuthResponse{}, err + } + chain, err := api.EventAuthForServer(ctx, t.from.name, roomID, eventID) + return federation.EventAuthResponse{AuthChain: chain}, httpError(err) +} + +func (t transport) StateIDs(ctx context.Context, dest, roomID, eventID string) (federation.StateIDsResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.StateIDsResponse{}, err + } + state, chain, err := api.StateIDsForServer(ctx, t.from.name, roomID, eventID) + return federation.StateIDsResponse{PDUIDs: state, AuthChainIDs: chain}, httpError(err) +} + +func (t transport) State(ctx context.Context, dest, roomID, eventID string) (federation.StateResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.StateResponse{}, err + } + state, chain, err := api.StateForServer(ctx, t.from.name, roomID, eventID) + return federation.StateResponse{PDUs: state, AuthChain: chain}, httpError(err) +} + +func (t transport) MissingEvents(ctx context.Context, dest, roomID string, req federation.MissingEventsRequest) (federation.MissingEventsResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.MissingEventsResponse{}, err + } + events, err := api.MissingEventsForServer(ctx, t.from.name, roomID, req.EarliestEvents, req.LatestEvents, req.Limit, req.MinDepth) + return federation.MissingEventsResponse{Events: events}, httpError(err) +} + +func (t transport) QueryDirectory(ctx context.Context, dest, alias string) (federation.DirectoryResponse, error) { + api, err := t.to(dest) + if err != nil { + return federation.DirectoryResponse{}, err + } + resolved, err := api.QueryAliasForServer(ctx, alias) + return federation.DirectoryResponse{RoomID: resolved.RoomID, Servers: resolved.Servers}, httpError(err) +} + +func (t transport) PublicRooms(ctx context.Context, dest string, limit int, since, search string) (json.RawMessage, error) { + api, err := t.to(dest) + if err != nil { + return nil, err + } + resp, err := api.PublicRooms(ctx, roomserver.PublicRoomsRequest{Limit: limit, Since: since, Search: search}) + if err != nil { + return nil, httpError(err) + } + return json.Marshal(resp) +} + +// --------------------------------------------------------------------------- + +const ( + serverA = "a.example" + serverB = "b.example" + serverC = "c.example" + + userA = "@alice:" + serverA + userA2 = "@anne:" + serverA + userB = "@bob:" + serverB + userC = "@cath:" + serverC +) + +// send sends a message and returns its event ID. +func (p *peer) send(t *testing.T, roomID, sender, body string) string { + t.Helper() + eventID, err := p.api.SendEvent(t.Context(), roomserver.SendEventRequest{ + RoomID: roomID, Sender: sender, Type: "m.room.message", + Content: json.RawMessage(`{"msgtype":"m.text","body":"` + body + `"}`), + }) + if err != nil { + t.Fatalf("%s: SendEvent: %v", p.name, err) + } + return eventID +} + +// timelineIDs returns the event IDs of a room's timeline on a server, +// oldest first. +func (p *peer) timelineIDs(t *testing.T, roomID string) []string { + t.Helper() + events, err := p.store.Timeline(t.Context(), storage.TimelineRequest{ + RoomID: roomID, Limit: 1000, Direction: storage.DirectionForward, + }) + if err != nil { + t.Fatalf("Timeline: %v", err) + } + ids := make([]string, 0, len(events)) + for _, e := range events { + ids = append(ids, e.EventID) + } + return ids +} + +// membership reads a user's membership on a server. +func (p *peer) membership(t *testing.T, roomID, userID string) string { + t.Helper() + m, err := p.api.Membership(t.Context(), roomID, userID) + if err != nil && !errors.Is(err, roomserver.ErrNoSuchRoom) { + t.Fatalf("Membership: %v", err) + } + return m +} + +// currentState returns a server's current state of a room as +// type/state_key -> event ID. +func (p *peer) currentState(t *testing.T, roomID string) map[string]string { + t.Helper() + entries, err := p.store.CurrentState(t.Context(), roomID) + if err != nil { + t.Fatalf("CurrentState: %v", err) + } + out := map[string]string{} + for _, e := range entries { + out[e.Type+"|"+e.StateKey] = e.EventID + } + return out +} + +// publicRoom creates a public room on a server. +func (p *peer) publicRoom(t *testing.T, creator string) string { + t.Helper() + resp, err := p.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{ + Creator: creator, Preset: roomserver.PresetPublicChat, Name: "Federated", + }) + if err != nil { + t.Fatalf("CreateRoom: %v", err) + } + return resp.RoomID +} + +func TestFederatedJoinAndConversation(t *testing.T) { + for _, version := range []string{"10", "11", "12"} { + t.Run("v"+version, func(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{ + Creator: userA, Preset: roomserver.PresetPublicChat, Name: "Federated", Version: version, + }) + if err != nil { + t.Fatalf("CreateRoom: %v", err) + } + roomID := resp.RoomID + a.takeOutbox() // nobody else is in the room yet + + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{ + RoomID: roomID, User: userB, ServerNames: []string{serverA}, + }); err != nil { + t.Fatalf("remote join: %v", err) + } + if got := b.membership(t, roomID, userB); got != event.MembershipJoin { + t.Fatalf("B sees its user as %q", got) + } + if got := a.membership(t, roomID, userB); got != event.MembershipJoin { + t.Fatalf("A sees B's user as %q", got) + } + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + + // Both servers hold the same room. + if sa, sb := a.currentState(t, roomID), b.currentState(t, roomID); !mapsEqual(sa, sb) { + t.Fatalf("current state differs:\nA: %v\nB: %v", sa, sb) + } + summary, err := b.store.RoomSummary(t.Context(), roomID) + if err != nil { + t.Fatalf("RoomSummary: %v", err) + } + if summary.Name != "Federated" || summary.JoinedMembers != 2 { + t.Errorf("B's listing of the room = %+v", summary.RoomProfile) + } + + // A conversation in both directions. + m1 := a.send(t, roomID, userA, "hello from A") + m2 := b.send(t, roomID, userB, "hello from B") + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + for _, p := range []*peer{a, b} { + ids := p.timelineIDs(t, roomID) + if !slices.Contains(ids, m1) || !slices.Contains(ids, m2) { + t.Errorf("%s's timeline lacks a message: %v", p.name, ids) + } + } + + // The two messages were sent concurrently, so each server now + // has two forward extremities; the next event joins them. + m3 := a.send(t, roomID, userA, "merge") + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + for _, p := range []*peer{a, b} { + exts, err := p.store.Extremities(t.Context(), roomID) + if err != nil { + t.Fatal(err) + } + if len(exts) != 1 || exts[0].EventID != m3 { + t.Errorf("%s's extremities after the merge = %v", p.name, eventIDsOf(exts)) + } + } + if sa, sb := a.currentState(t, roomID), b.currentState(t, roomID); !mapsEqual(sa, sb) { + t.Errorf("current state differs after the conversation:\nA: %v\nB: %v", sa, sb) + } + }) + } +} + +func TestRemoteJoinFailsOverToTheNextServer(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{ + RoomID: roomID, User: userB, ServerNames: []string{"gone.example", serverA}, + }); err != nil { + t.Fatalf("join with an unreachable first server: %v", err) + } + if got := b.membership(t, roomID, userB); got != event.MembershipJoin { + t.Errorf("membership = %q", got) + } +} + +func TestRemoteJoinToAnInviteOnlyRoomIsRefused(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{Creator: userA, Preset: roomserver.PresetPrivateChat}) + if err != nil { + t.Fatal(err) + } + + _, err = b.api.JoinRoom(t.Context(), roomserver.JoinRequest{ + RoomID: resp.RoomID, User: userB, ServerNames: []string{serverA}, + }) + if !errors.Is(err, roomserver.ErrForbidden) { + t.Fatalf("join to an invite-only room = %v, want ErrForbidden", err) + } + if _, err := b.store.Room(t.Context(), resp.RoomID); !errors.Is(err, storage.ErrNoSuchRoom) { + t.Errorf("a refused join left the room behind on B: %v", err) + } +} + +func TestRemoteInviteJoinAndReject(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{ + Creator: userA, Preset: roomserver.PresetPrivateChat, Name: "Secret", + }) + if err != nil { + t.Fatal(err) + } + roomID := resp.RoomID + + if err := a.api.InviteUser(t.Context(), roomserver.InviteRequest{RoomID: roomID, Sender: userA, Target: userB}); err != nil { + t.Fatalf("remote invite: %v", err) + } + if got := b.membership(t, roomID, userB); got != event.MembershipInvite { + t.Fatalf("B sees its user as %q after the invite", got) + } + // The invite on A carries both servers' signatures. + stored, err := a.store.CurrentStateEvent(t.Context(), roomID, event.TypeMember, userB) + if err != nil { + t.Fatal(err) + } + var sigs struct { + Signatures map[string]any `json:"signatures"` + } + _ = json.Unmarshal(stored.JSON, &sigs) + if _, ok := sigs.Signatures[serverB]; !ok { + t.Error("the invite A holds is not countersigned by B") + } + + // B shows the invite with the state A sent along. + stripped, err := b.api.StrippedState(t.Context(), roomID, []string{userB}) + if err != nil { + t.Fatalf("StrippedState: %v", err) + } + var sawName, sawInvite bool + for _, s := range stripped { + if s.Type == "m.room.name" { + sawName = true + } + if s.Type == event.TypeMember && s.StateKey == userB { + sawInvite = true + } + } + if !sawName || !sawInvite { + t.Errorf("B's invite state = %+v", stripped) + } + + // Accepting goes through A, which is the inviter. + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB}); err != nil { + t.Fatalf("accepting the invite: %v", err) + } + if got := a.membership(t, roomID, userB); got != event.MembershipJoin { + t.Errorf("A sees B's user as %q after the join", got) + } + if got, _ := b.store.InviteState(t.Context(), roomID, userB); got != nil { + t.Error("the invite state outlived the invite") + } + + // B is in the room now, so a second invite to one of its users is + // taken in as an ordinary event of the room rather than out of band. + m.flush() + if err := a.api.InviteUser(t.Context(), roomserver.InviteRequest{RoomID: roomID, Sender: userA, Target: "@zed:" + serverB}); err != nil { + t.Fatalf("second invite: %v", err) + } + m.flush() + if got := b.membership(t, roomID, "@zed:"+serverB); got != event.MembershipInvite { + t.Fatalf("B sees zed as %q", got) + } +} + +func TestRejectingARemoteInvite(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{Creator: userA, Preset: roomserver.PresetPrivateChat}) + if err != nil { + t.Fatal(err) + } + roomID := resp.RoomID + if err := a.api.InviteUser(t.Context(), roomserver.InviteRequest{RoomID: roomID, Sender: userA, Target: userB}); err != nil { + t.Fatal(err) + } + + if err := b.api.LeaveRoom(t.Context(), roomID, userB, "no thanks"); err != nil { + t.Fatalf("rejecting: %v", err) + } + if got := b.membership(t, roomID, userB); got != event.MembershipLeave { + t.Errorf("B sees its user as %q after rejecting", got) + } + if got := a.membership(t, roomID, userB); got != event.MembershipLeave { + t.Errorf("A sees B's user as %q after the rejection", got) + } +} + +func TestRejectingAnInviteFromAnUnreachableServer(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{Creator: userA, Preset: roomserver.PresetPrivateChat}) + if err != nil { + t.Fatal(err) + } + if err := a.api.InviteUser(t.Context(), roomserver.InviteRequest{RoomID: resp.RoomID, Sender: userA, Target: userB}); err != nil { + t.Fatal(err) + } + delete(m.servers, serverA) + + if err := b.api.LeaveRoom(t.Context(), resp.RoomID, userB, ""); err != nil { + t.Fatalf("rejecting locally: %v", err) + } + if got := b.membership(t, resp.RoomID, userB); got != event.MembershipLeave { + t.Errorf("membership after a local rejection = %q", got) + } +} + +func TestMissingEventsAreFetched(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); err != nil { + t.Fatal(err) + } + m.flush() + + var sent []string + for i := range 4 { + sent = append(sent, a.send(t, roomID, userA, fmt.Sprint("m", i))) + } + outbox := a.takeOutbox() + // Only the last one gets through; B has to ask for the other three. + last := outbox[len(outbox)-1] + if err := m.deliverOnly(serverA, serverB, last.Event); err != nil { + t.Fatalf("delivering the last message: %v", err) + } + ids := b.timelineIDs(t, roomID) + for _, id := range sent { + if !slices.Contains(ids, id) { + t.Errorf("B's timeline lacks %s", id) + } + } + // In order. + var positions []int + for _, id := range sent { + positions = append(positions, slices.Index(ids, id)) + } + if !slices.IsSorted(positions) { + t.Errorf("fetched events are out of order: %v", positions) + } +} + +func TestAWideGapIsClosedWithState(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); err != nil { + t.Fatal(err) + } + m.flush() + + // More than get_missing_events will be asked for, with a state change + // in the part that is not fetched. + for i := range 6 { + a.send(t, roomID, userA, fmt.Sprint("early", i)) + } + if _, err := a.api.SendEvent(t.Context(), roomserver.SendEventRequest{ + RoomID: roomID, Sender: userA, Type: "m.room.topic", StateKey: ptrTo(""), + Content: json.RawMessage(`{"topic":"changed in the gap"}`), + }); err != nil { + t.Fatal(err) + } + for i := range 15 { + a.send(t, roomID, userA, fmt.Sprint("late", i)) + } + outbox := a.takeOutbox() + last := outbox[len(outbox)-1] + if err := m.deliverOnly(serverA, serverB, last.Event); err != nil { + t.Fatalf("delivering across the gap: %v", err) + } + + if !slices.Contains(b.timelineIDs(t, roomID), last.Event.EventID()) { + t.Error("the event across the gap is not in B's timeline") + } + if sa, sb := a.currentState(t, roomID), b.currentState(t, roomID); !mapsEqual(sa, sb) { + t.Errorf("current state differs after the gap:\nA: %v\nB: %v", sa, sb) + } +} + +func TestForgedEventIsRejected(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); err != nil { + t.Fatal(err) + } + m.flush() + + // An event in alice's name, signed by a key that is not A's. + exts, err := b.store.Extremities(t.Context(), roomID) + if err != nil { + t.Fatal(err) + } + impostor, _ := signing.Generate("k_a") + room, _ := b.store.Room(t.Context(), roomID) + rules := mustRules(t, room.Version) + state := b.currentState(t, roomID) + authEvents := []string{state[event.TypePowerLevels+"|"], state[event.TypeMember+"|"+userA]} + if !rules.CreateEventExcludedFromAuthEvents { + authEvents = append(authEvents, state[event.TypeCreate+"|"]) + } + forged, err := (&event.Builder{ + RoomID: roomID, Type: "m.room.message", Sender: userA, + Content: json.RawMessage(`{"body":"not really alice"}`), + PrevEvents: []string{exts[0].EventID}, AuthEvents: authEvents, Depth: exts[0].Depth + 1, + }).Build(fixedNow(), impostor, rules) + if err != nil { + t.Fatal(err) + } + if err := m.deliverOnly(serverA, serverB, forged); !errors.Is(err, roomserver.ErrBadEvent) { + t.Errorf("a forged event = %v, want ErrBadEvent", err) + } + if slices.Contains(b.timelineIDs(t, roomID), forged.EventID()) { + t.Error("the forged event is in the timeline") + } +} + +func TestEventFailingOnlyAgainstCurrentStateIsSoftFailed(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); err != nil { + t.Fatal(err) + } + m.flush() + + // B's user speaks, and the message is held back; meanwhile A bans + // them, and the ban reaches B. + b.send(t, roomID, userB, "last words") + late := b.takeOutbox()[0].Event + if err := a.api.BanUser(t.Context(), roomID, userA, userB, "spam"); err != nil { + t.Fatalf("BanUser: %v", err) + } + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + + // The message was valid when it was sent, and is not now. + if err := m.deliverOnly(serverB, serverA, late); err != nil { + t.Fatalf("a soft-failed event is not an error to its sender: %v", err) + } + stored, err := a.store.Event(t.Context(), roomID, late.EventID()) + if err != nil { + t.Fatalf("the soft-failed event was not kept: %v", err) + } + if !stored.Outlier { + t.Error("the soft-failed event is not an outlier") + } + if slices.Contains(a.timelineIDs(t, roomID), late.EventID()) { + t.Error("the soft-failed event is in A's timeline") + } +} + +func TestFederationSwitchRefusesRemoteWork(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + + m.setDisabled(true) + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); !errors.Is(err, roomserver.ErrFederationDisabled) { + t.Errorf("remote join while disabled = %v, want ErrFederationDisabled", err) + } + if err := a.api.InviteUser(t.Context(), roomserver.InviteRequest{RoomID: roomID, Sender: userA, Target: userB}); !errors.Is(err, roomserver.ErrFederationDisabled) { + t.Errorf("remote invite while disabled = %v, want ErrFederationDisabled", err) + } + if _, _, err := a.api.MakeJoin(t.Context(), serverB, roomID, userB, []string{"11"}); !errors.Is(err, roomserver.ErrFederationDisabled) { + t.Errorf("make_join while disabled = %v, want ErrFederationDisabled", err) + } + + m.setDisabled(false) + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); err != nil { + t.Errorf("remote join after re-enabling: %v", err) + } +} + +func TestThreeServersAgree(t *testing.T) { + m := newMesh(t, serverA, serverB, serverC) + a, b, c := m.servers[serverA], m.servers[serverB], m.servers[serverC] + roomID := a.publicRoom(t, userA) + + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); err != nil { + t.Fatal(err) + } + m.flush() + // C joins through B, which is not the room's creator: B relays the + // join to A. + if _, err := c.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userC, ServerNames: []string{serverB}}); err != nil { + t.Fatalf("C joining through B: %v", err) + } + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + if got := a.membership(t, roomID, userC); got != event.MembershipJoin { + t.Fatalf("A did not hear of C's join through B's relay: %q", got) + } + + a.send(t, roomID, userA2Join(t, a, roomID), "from A") + b.send(t, roomID, userB, "from B") + c.send(t, roomID, userC, "from C") + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + final := a.send(t, roomID, userA, "settle") + m.flush() + + sa, sb, sc := a.currentState(t, roomID), b.currentState(t, roomID), c.currentState(t, roomID) + if !mapsEqual(sa, sb) || !mapsEqual(sa, sc) { + t.Errorf("the three servers disagree:\nA: %v\nB: %v\nC: %v", sa, sb, sc) + } + for _, p := range []*peer{a, b, c} { + if !slices.Contains(p.timelineIDs(t, roomID), final) { + t.Errorf("%s lacks the final message", p.name) + } + } +} + +// userA2Join joins a second local user on A, so that local joins in a +// federated room are covered too, and returns them. +func userA2Join(t *testing.T, a *peer, roomID string) string { + t.Helper() + if _, err := a.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userA2}); err != nil { + t.Fatalf("local join in a federated room: %v", err) + } + return userA2 +} + +func TestRemoteAliasAndDirectory(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{ + Creator: userA, Preset: roomserver.PresetPublicChat, Visibility: roomserver.VisibilityPublic, + RoomAliasName: "lobby", Name: "Lobby", + }) + if err != nil { + t.Fatal(err) + } + + resolved, err := b.api.ResolveAlias(t.Context(), "#lobby:"+serverA) + if err != nil { + t.Fatalf("ResolveAlias: %v", err) + } + if resolved.RoomID != resp.RoomID || !slices.Contains(resolved.Servers, serverA) { + t.Errorf("ResolveAlias = %+v", resolved) + } + if _, err := b.api.ResolveAlias(t.Context(), "#nothing:"+serverA); !errors.Is(err, roomserver.ErrNoSuchAlias) { + t.Errorf("unknown remote alias = %v, want ErrNoSuchAlias", err) + } + + dir, err := b.api.PublicRooms(t.Context(), roomserver.PublicRoomsRequest{Server: serverA}) + if err != nil { + t.Fatalf("PublicRooms: %v", err) + } + if len(dir.Chunk) != 1 || dir.Chunk[0].RoomID != resp.RoomID || dir.Chunk[0].Name != "Lobby" { + t.Errorf("remote directory = %+v", dir) + } +} + +func TestServersOutsideARoomCannotReadIt(t *testing.T) { + m := newMesh(t, serverA, serverB) + a := m.servers[serverA] + resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{Creator: userA, Preset: roomserver.PresetPrivateChat}) + if err != nil { + t.Fatal(err) + } + exts, err := a.store.Extremities(t.Context(), resp.RoomID) + if err != nil { + t.Fatal(err) + } + if _, err := a.api.EventForServer(t.Context(), serverB, exts[0].EventID); !errors.Is(err, roomserver.ErrForbidden) { + t.Errorf("/event from outside = %v, want ErrForbidden", err) + } + if _, _, err := a.api.StateIDsForServer(t.Context(), serverB, resp.RoomID, exts[0].EventID); !errors.Is(err, roomserver.ErrForbidden) { + t.Errorf("/state_ids from outside = %v, want ErrForbidden", err) + } + if _, err := a.api.BackfillForServer(t.Context(), serverB, resp.RoomID, []string{exts[0].EventID}, 10); !errors.Is(err, roomserver.ErrForbidden) { + t.Errorf("/backfill from outside = %v, want ErrForbidden", err) + } +} + +func TestBackfillWalksBackwards(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + roomID := a.publicRoom(t, userA) + if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: userB, ServerNames: []string{serverA}}); err != nil { + t.Fatal(err) + } + var sent []string + for i := range 5 { + sent = append(sent, a.send(t, roomID, userA, fmt.Sprint(i))) + } + events, err := a.api.BackfillForServer(t.Context(), serverB, roomID, []string{sent[4]}, 3) + if err != nil { + t.Fatalf("BackfillForServer: %v", err) + } + if len(events) != 3 { + t.Fatalf("backfill returned %d events, want 3", len(events)) + } + var got []string + for _, raw := range events { + var probe struct { + Content struct { + Body string `json:"body"` + } `json:"content"` + } + _ = json.Unmarshal(raw, &probe) + got = append(got, probe.Content.Body) + } + if !slices.Equal(got, []string{"4", "3", "2"}) { + t.Errorf("backfill bodies = %v, want newest first from the starting event", got) + } +} + +func mapsEqual(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} + +func eventIDsOf(events []storage.StoredEvent) []string { + out := make([]string, 0, len(events)) + for _, e := range events { + out = append(out, e.EventID) + } + return out +} + +func ptrTo[T any](v T) *T { return &v } + +func mustRules(t *testing.T, version string) roomversion.Rules { + t.Helper() + rules, err := roomversion.Get(roomversion.Version(version)) + if err != nil { + t.Fatal(err) + } + return rules +} + +func fixedNow() time.Time { return time.Unix(1_700_000_000, 0) } + +func TestServerACLKeepsAServerOut(t *testing.T) { + m := newMesh(t, serverA, serverB, serverC) + a, b, c := m.servers[serverA], m.servers[serverB], m.servers[serverC] + roomID := a.publicRoom(t, userA) + for _, p := range []*peer{b, c} { + user := userB + if p == c { + user = userC + } + if _, err := p.api.JoinRoom(t.Context(), roomserver.JoinRequest{RoomID: roomID, User: user, ServerNames: []string{serverA}}); err != nil { + t.Fatal(err) + } + } + m.flush() + + if _, err := a.api.SendEvent(t.Context(), roomserver.SendEventRequest{ + RoomID: roomID, Sender: userA, Type: roomserver.TypeServerACL, StateKey: ptrTo(""), + Content: json.RawMessage(`{"allow":["*"],"deny":["` + serverC + `"]}`), + }); err != nil { + t.Fatalf("setting the ACL: %v", err) + } + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + + c.send(t, roomID, userC, "from a denied server") + failures := m.flush() + if len(failures) == 0 { + t.Fatal("the denied server's event was accepted") + } + for _, err := range failures { + if !errors.Is(err, roomserver.ErrForbidden) { + t.Errorf("refusal = %v, want ErrForbidden", err) + } + } + if _, _, err := a.api.MakeJoin(t.Context(), serverC, roomID, "@other:"+serverC, []string{"11", "12", "10"}); !errors.Is(err, roomserver.ErrForbidden) { + t.Errorf("make_join from a denied server = %v, want ErrForbidden", err) + } + + // B, which the ACL allows, is unaffected. + b.send(t, roomID, userB, "from an allowed server") + if failures := m.flush(); len(failures) > 0 { + t.Errorf("an allowed server's event was refused: %v", failures) + } +} + +func (m *mesh) setDisabled(disabled bool) { + m.mu.Lock() + defer m.mu.Unlock() + m.disabled = disabled +} diff --git a/server/internal/roomserver/input.go b/server/internal/roomserver/input.go index 64d44a9..4d0956a 100644 --- a/server/internal/roomserver/input.go +++ b/server/internal/roomserver/input.go @@ -41,6 +41,14 @@ type sendRequest struct { // Transaction, when set, records the client transaction ID this event // came from, so that a retried request does not send a second event. Transaction *storage.TransactionRef + + // countersign, when set, is given the event once it has been built and + // authorised and returns the copy to persist. An invite to a user of + // another server goes through it: that server has to sign the invite + // before it can be in the room, and the event persisted is the one with + // both signatures. It runs under the room lock, so the event it signs + // is still a child of the room's current extremities when it lands. + countersign func(context.Context, *event.Event) (*event.Event, error) } // send is the input pipeline for an event this server creates. @@ -125,11 +133,17 @@ func (r *API) send(ctx context.Context, req sendRequest) (*event.Event, error) { // sees this event. For an event created here that state is also the // room's current state, so there is no separate soft-failure check to // make: the two can only differ for an event that arrives from - // somewhere else, which is Phase 5. + // somewhere else, and federated.go checks for that as soft failure. if err := auth.Check(ctx, built, checkState(authEvents, st), r.verifier()); err != nil { return nil, err } + if req.countersign != nil { + if built, err = req.countersign(ctx, built); err != nil { + return nil, err + } + } + if err := r.persist(ctx, persistRequest{ Room: room, Rules: rules, @@ -227,10 +241,9 @@ func nextDepth(prevEvents []*event.Event) int64 { // resolved state has to be written out whole: tuples the resolution dropped // have to go, and an incremental update cannot express that. // -// Only federation forks a room, so in Phase 3 the second path is unreachable -// from outside. It is here rather than in Phase 5 because the alternative is -// a pipeline that authorises against one arbitrary branch of a fork without -// saying so. +// Only federation forks a room. The second path was written in Phase 3, +// before anything could reach it, because the alternative was a pipeline +// that authorised against one arbitrary branch of a fork without saying so. func (r *API) roomState(ctx context.Context, room storage.Room, rules roomversion.Rules) (auth.State, []*event.Event, bool, error) { storedExtremities, err := r.store.Extremities(ctx, room.ID) if err != nil { @@ -312,6 +325,22 @@ type persistRequest struct { Transaction *storage.TransactionRef RedactsEventID string + + // Extremities replaces the room's forward extremities. Nil means the + // event alone, which is right for every event created here: it + // descends from every extremity there was. + Extremities []string + + // CurrentState, when set, is the room's whole current state after the + // event, written as a replacement. An event from another server sets + // it whenever the room's state after it is not simply the state before + // it plus the event: when it forks the room, or when it was built on + // something other than the room's single extremity. + CurrentState auth.State + + // Origin and Relay are passed to the notifier; see Notification. + Origin string + Relay bool } // persist writes the event and everything its arrival changes, then emits. @@ -330,9 +359,12 @@ func (r *API) persist(ctx context.Context, req persistRequest) error { JSON: e.JSON(), }, StateBefore: stateEntries(req.StateBefore), - Extremities: []string{e.EventID()}, + Extremities: req.Extremities, Transaction: req.Transaction, } + if write.Extremities == nil { + write.Extremities = []string{e.EventID()} + } if isState { write.Event.StateKey = &stateKey } @@ -341,11 +373,19 @@ func (r *API) persist(ctx context.Context, req persistRequest) error { // to upsert; after a resolution it is the whole state, which the store // is told to write as a replacement. stateAfter := req.StateBefore - if req.ReplaceWholeState { + stateChanged := isState + switch { + case req.CurrentState != nil: + write.StateDeltaIsWholeState = true + write.StateDelta = stateEntries(req.CurrentState) + stateAfter = req.CurrentState + stateChanged = true + case req.ReplaceWholeState: write.StateDeltaIsWholeState = true write.StateDelta = stateEntries(req.StateBefore) + stateChanged = true } - if isState { + if isState && req.CurrentState == nil { entry := stateEntry(e.Type(), stateKey, e) if req.ReplaceWholeState { write.StateDelta = upsertEntry(write.StateDelta, entry) @@ -357,8 +397,9 @@ func (r *API) persist(ctx context.Context, req persistRequest) error { // The denormalised listing columns are recomputed only when this event // is one of the ones behind them, which keeps a room full of messages - // from rewriting its own row on every line of chat. - if isState && affectsRoomProfile(e.Type()) { + // from rewriting its own row on every line of chat. A whole-state + // write may have changed any of them, so it always recomputes. + if (isState && affectsRoomProfile(e.Type())) || write.StateDeltaIsWholeState { profile := roomProfile(stateAfter) write.Profile = &profile } @@ -383,7 +424,9 @@ func (r *API) persist(ctx context.Context, req persistRequest) error { RoomID: req.Room.ID, Event: e, StreamOrdering: ordering, - StateChanged: isState, + StateChanged: stateChanged, + Origin: req.Origin, + Relay: req.Relay, }) } return nil diff --git a/server/internal/roomserver/membership.go b/server/internal/roomserver/membership.go index aa4d1b1..f6462ad 100644 --- a/server/internal/roomserver/membership.go +++ b/server/internal/roomserver/membership.go @@ -30,31 +30,25 @@ type JoinRequest struct { ServerNames []string } -// JoinRoom joins a local room. +// JoinRoom joins a room. // -// Joining a room this server does not have is a federated join: the whole -// room has to be fetched from a server that does have it, which is Phase 5. -// While federation is off that is refused with a message saying so, which is -// the behaviour PLAN.md section 3 asks for — a client otherwise sees a bare +// Joining a room this server is not in is a federated join: the room has +// to be fetched from a server that is (see federated_membership.go). While +// federation is off that is refused with a message saying so, which is the +// behaviour PLAN.md section 3 asks for — a client otherwise sees a bare // M_FORBIDDEN and the operator goes looking through power levels. func (r *API) JoinRoom(ctx context.Context, req JoinRequest) (string, error) { if !r.isLocal(req.User) { return "", fmt.Errorf("%w: %s is not a user of this server", ErrForbidden, req.User) } - switch _, err := r.store.Room(ctx, req.RoomID); { - case errors.Is(err, ErrNoSuchRoom): - if !r.federationEnabled() { - return "", fmt.Errorf("%w: %s is not a room on this server, and joining a remote room", - ErrFederationDisabled, req.RoomID) - } - // With federation on, a room we do not have still has to be - // fetched from a server that does. That is Phase 5. - return "", fmt.Errorf("%w: joining a room this server does not have is not implemented yet", - ErrFederationDisabled) - case err != nil: + remote, err := r.joinNeedsFederation(ctx, req) + if err != nil { return "", err } + if remote { + return r.joinRemote(ctx, req) + } // Joining a room you are already in is a no-op rather than a second // event: clients call /join to make sure, and every extra membership @@ -241,7 +235,31 @@ func (r *API) localUserWhoCanInvite(st auth.State, roomID string) (string, error // the user themselves. Whether they were joined, invited or knocking is the // authorisation rules' business, and a user who was never in the room is // refused by them. +// +// A room this server is not in — one it only holds an invite to — cannot +// take an event from here, and the leave goes through a server that is in +// it instead. func (r *API) LeaveRoom(ctx context.Context, roomID, userID, reason string) error { + if r.isLocal(userID) { + if _, err := r.store.Room(ctx, roomID); err != nil { + return err + } + if _, err := r.store.CurrentStateEvent(ctx, roomID, event.TypeCreate, ""); isNotFound(err) { + return r.leaveRemote(ctx, roomID, userID, reason) + } else if err != nil { + return err + } + in, err := r.participating(ctx, roomID) + if err != nil { + return err + } + if !in { + if membership, err := r.store.Membership(ctx, roomID, userID); err == nil && + membership.Membership == event.MembershipInvite && !r.isLocalEvent(ctx, roomID, membership.EventID) { + return r.leaveRemote(ctx, roomID, userID, reason) + } + } + } _, err := r.send(ctx, sendRequest{ RoomID: roomID, Type: event.TypeMember, @@ -279,12 +297,7 @@ func (r *API) InviteUser(ctx context.Context, req InviteRequest) error { return err } if !r.isLocal(req.Target) { - if !r.federationEnabled() { - return fmt.Errorf("%w: %s is on another server, and inviting a remote user", - ErrFederationDisabled, req.Target) - } - return fmt.Errorf("%w: inviting a user of another server is not implemented yet", - ErrFederationDisabled) + return r.inviteRemote(ctx, req) } _, err := r.send(ctx, sendRequest{ @@ -457,3 +470,10 @@ func validateUserID(userID string) error { } return nil } + +// isLocalEvent reports whether an event of a room was sent by a user of +// this server. +func (r *API) isLocalEvent(ctx context.Context, roomID, eventID string) bool { + stored, err := r.store.Event(ctx, roomID, eventID) + return err == nil && r.isLocal(stored.Sender) +} diff --git a/server/internal/roomserver/remote.go b/server/internal/roomserver/remote.go new file mode 100644 index 0000000..93d737f --- /dev/null +++ b/server/internal/roomserver/remote.go @@ -0,0 +1,335 @@ +package roomserver + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "slices" + "sort" + + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/auth" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" + "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// RemoteServers is what the roomserver asks of other servers. +// +// *federation.Client satisfies it. It is an interface so that the +// roomserver's federation paths can be tested against a fake server in +// the same process, and so that the roomserver depends on the shape of the +// federation API rather than on how requests are signed and routed. +type RemoteServers interface { + MakeJoin(ctx context.Context, destination, roomID, userID string, versions []string) (federation.MakeJoinResponse, error) + SendJoin(ctx context.Context, destination, roomID, eventID string, joinEvent json.RawMessage) (federation.SendJoinResponse, error) + MakeLeave(ctx context.Context, destination, roomID, userID string) (federation.MakeJoinResponse, error) + SendLeave(ctx context.Context, destination, roomID, eventID string, leaveEvent json.RawMessage) error + Invite(ctx context.Context, destination, roomID, eventID string, req federation.InviteRequest) (federation.InviteResponse, error) + Event(ctx context.Context, destination, eventID string) (federation.Transaction, error) + EventAuth(ctx context.Context, destination, roomID, eventID string) (federation.EventAuthResponse, error) + StateIDs(ctx context.Context, destination, roomID, eventID string) (federation.StateIDsResponse, error) + State(ctx context.Context, destination, roomID, eventID string) (federation.StateResponse, error) + MissingEvents(ctx context.Context, destination, roomID string, req federation.MissingEventsRequest) (federation.MissingEventsResponse, error) + Backfill(ctx context.Context, destination, roomID string, from []string, limit int) (federation.Transaction, error) + QueryDirectory(ctx context.Context, destination, alias string) (federation.DirectoryResponse, error) + PublicRooms(ctx context.Context, destination string, limit int, since, search string) (json.RawMessage, error) +} + +// federating reports whether this server may talk to others right now. +func (r *API) federating() bool { + return r.remote != nil && r.keys != nil && r.federationEnabled() +} + +// requireFederation is the error for an operation that needs another +// server: the switch is off, or this build has no federation at all. +func (r *API) requireFederation(action string) error { + if r.federating() { + return nil + } + return fmt.Errorf("%w: %s", ErrFederationDisabled, action) +} + +// participating reports whether this server is in a room: it holds the +// room's state, and at least one of its users is joined. +// +// Holding the room row is not enough. An invite to a room nobody here is +// in leaves a row with one membership and no create event, and a room +// every local user has left is a room whose state stopped being kept up +// to date the moment the last of them went. +func (r *API) participating(ctx context.Context, roomID string) (bool, error) { + if _, err := r.store.CurrentStateEvent(ctx, roomID, event.TypeCreate, ""); err != nil { + if isNotFound(err) { + return false, nil + } + return false, err + } + return r.store.HasJoinedMemberOn(ctx, roomID, r.serverName) +} + +// JoinedServers lists the servers with a user joined to a room, this one +// included when it is, sorted. It is who an event sent into the room is +// delivered to. +func (r *API) JoinedServers(ctx context.Context, roomID string) ([]string, error) { + joined, err := r.store.MembershipEntries(ctx, roomID, []string{event.MembershipJoin}) + if err != nil { + return nil, err + } + seen := map[string]bool{} + for _, m := range joined { + if server, err := id.ServerNameOf(m.UserID); err == nil { + seen[server] = true + } + } + out := make([]string, 0, len(seen)) + for server := range seen { + out = append(out, server) + } + sort.Strings(out) + return out, nil +} + +// ServerInRoom reports whether a server has a user joined to a room. +func (r *API) ServerInRoom(ctx context.Context, roomID, serverName string) (bool, error) { + return r.store.HasJoinedMemberOn(ctx, roomID, serverName) +} + +// candidateServers orders the servers worth asking about a room: the ones +// the caller suggested, then the one named in the room ID, then the +// servers of the room's members as this server last knew them. This +// server is never among them. +func (r *API) candidateServers(ctx context.Context, roomID string, hints []string) []string { + var out []string + add := func(server string) { + if server != "" && server != r.serverName && !slices.Contains(out, server) { + out = append(out, server) + } + } + for _, h := range hints { + add(h) + } + if server, err := id.ServerNameOfRoomID(roomID); err == nil { + add(server) + } + if members, err := r.store.MembershipEntries(ctx, roomID, nil); err == nil { + // Joined members first: they are in the room, and an invited or + // departed one may not be. + for _, m := range members { + if m.Membership == event.MembershipJoin { + if server, err := id.ServerNameOf(m.UserID); err == nil { + add(server) + } + } + } + for _, m := range members { + if server, err := id.ServerNameOf(m.UserID); err == nil { + add(server) + } + // The sender of an invite to a local user is who brought this + // server to the room, and is the server most likely to let it + // in. + if m.Membership == event.MembershipInvite && r.isLocal(m.UserID) { + if stored, err := r.store.Event(ctx, roomID, m.EventID); err == nil { + if server, err := id.ServerNameOf(stored.Sender); err == nil { + add(server) + } + } + } + } + } + return out +} + +// remoteError translates a failure from another server into the +// roomserver's own sentinels, so that a client hears "forbidden" when the +// remote server said forbidden, rather than "a remote server failed". +func remoteError(server string, err error) error { + var httpErr *federation.HTTPError + if errors.As(err, &httpErr) { + switch { + case httpErr.Code == "M_INCOMPATIBLE_ROOM_VERSION": + return fmt.Errorf("%w: %s: %s", ErrIncompatibleRoomVersion, server, httpErr.Message) + case httpErr.Status == http.StatusForbidden: + return fmt.Errorf("%w: %s refused: %s", ErrForbidden, server, httpErr.Message) + case httpErr.Status == http.StatusNotFound: + return fmt.Errorf("%w: %s does not know the room: %s", ErrNoSuchRoom, server, httpErr.Message) + } + } + switch { + case errors.Is(err, federation.ErrDisabled): + return fmt.Errorf("%w: %v", ErrFederationDisabled, err) + case errors.Is(err, federation.ErrNotAllowed): + return fmt.Errorf("%w: %v", ErrForbidden, err) + } + return fmt.Errorf("%w: %s: %v", ErrRemote, server, err) +} + +// parseRemote parses an event received from another server under the +// rules of the room it claims to belong to. +func parseRemote(raw json.RawMessage, rules roomversion.Rules) (*event.Event, error) { + e, err := event.Parse(stripUnsigned(raw), rules) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrBadEvent, err) + } + return e, nil +} + +// stripUnsigned drops the unsigned block of an event received from another +// server. It is that server's annotation — age, the state it replaced, the +// invite room state — and anything kept from it would be served to our +// own clients as if this server had written it. +func stripUnsigned(raw json.RawMessage) json.RawMessage { + var obj map[string]json.RawMessage + if err := json.Unmarshal(raw, &obj); err != nil { + return raw + } + if _, ok := obj["unsigned"]; !ok { + return raw + } + delete(obj, "unsigned") + out, err := json.Marshal(obj) + if err != nil { + return raw + } + return out +} + +// roomIDOfRaw reads the room_id of an event that has not been parsed yet, +// which is what decides the rules it will be parsed under. +func roomIDOfRaw(raw json.RawMessage) (string, error) { + var probe struct { + RoomID string `json:"room_id"` + } + if err := json.Unmarshal(raw, &probe); err != nil { + return "", fmt.Errorf("%w: %v", ErrBadEvent, err) + } + if probe.RoomID == "" { + return "", fmt.Errorf("%w: no room_id", ErrBadEvent) + } + return probe.RoomID, nil +} + +// storedOf projects an event onto the row the store keeps. +func storedOf(roomID string, e *event.Event) storage.StoredEvent { + s := storage.StoredEvent{ + RoomID: roomID, + EventID: e.EventID(), + Type: e.Type(), + Sender: e.Sender(), + Depth: e.Depth(), + OriginServerTS: e.OriginServerTS(), + JSON: e.JSON(), + } + if key, ok := e.StateKey(); ok { + s.StateKey = &key + } + return s +} + +// supportedVersionStrings lists the room versions Warp can join, for the +// ?ver= parameters of make_join. +func supportedVersionStrings() []string { + versions := roomversion.Supported() + out := make([]string, 0, len(versions)) + for _, v := range versions { + out = append(out, string(v)) + } + return out +} + +// eventsByDepth sorts events oldest first, with the event ID as the tie +// break so that the order is the same on every run. +func eventsByDepth(events []*event.Event) { + sort.SliceStable(events, func(i, j int) bool { + if events[i].Depth() != events[j].Depth() { + return events[i].Depth() < events[j].Depth() + } + return events[i].EventID() < events[j].EventID() + }) +} + +// authCheckChain checks a set of events from another server against their +// own auth events, oldest first, and returns the ones that pass. +// +// It is how a room's state and auth chain, as handed over by a join or a +// /state fetch, is judged: nothing in it is trusted because the resident +// server sent it, and an event whose auth events failed fails with them. +// known supplies auth events from outside the set — ones this server +// already holds. +func (r *API) authCheckChain(ctx context.Context, events []*event.Event, known map[string]*event.Event) (map[string]*event.Event, []string) { + eventsByDepth(events) + passed := map[string]*event.Event{} + var rejected []string + + lookup := func(id string) *event.Event { + if e, ok := passed[id]; ok { + return e + } + return known[id] + } + + // The create event is found first: from room version 12 nobody cites + // it, and every rule needs it. + var create *event.Event + for _, e := range events { + if e.Type() == event.TypeCreate { + create = e + break + } + } + if create == nil { + for _, e := range known { + if e.Type() == event.TypeCreate { + create = e + break + } + } + } + + for _, e := range events { + var authEvents []*event.Event + missing := false + for _, id := range e.AuthEventIDs() { + ae := lookup(id) + if ae == nil { + missing = true + break + } + authEvents = append(authEvents, ae) + } + if missing { + rejected = append(rejected, e.EventID()) + continue + } + if err := auth.CheckAuthEvents(e, authEvents); err != nil { + rejected = append(rejected, e.EventID()) + continue + } + st := auth.NewState(authEvents) + if st.Create() == nil && create != nil { + st[auth.StateKey{Type: event.TypeCreate}] = create + } + if err := auth.Check(ctx, e, st, r.verifier()); err != nil { + r.log.Debug("event in a received chain fails authorisation", + "event_id", e.EventID(), "type", e.Type(), "error", err) + rejected = append(rejected, e.EventID()) + continue + } + passed[e.EventID()] = e + } + return passed, rejected +} + +// inRoom reports whether an event belongs to a room. A create event of +// room version 12 carries no room_id; it belongs to the room its own ID +// names. +func inRoom(e *event.Event, roomID string) bool { + if e.Type() == event.TypeCreate && e.Rules().RoomIDIsCreateEventID { + id, err := event.RoomIDOfCreateEvent(e) + return err == nil && id == roomID + } + return e.RoomID() == roomID +} diff --git a/server/internal/roomserver/roomserver.go b/server/internal/roomserver/roomserver.go index 211ebb7..01e6a29 100644 --- a/server/internal/roomserver/roomserver.go +++ b/server/internal/roomserver/roomserver.go @@ -66,6 +66,23 @@ var ( // ErrUnknownRoomVersion is a room version that does not exist. ErrUnknownRoomVersion = roomversion.ErrUnknownVersion + + // ErrIncompatibleRoomVersion is a federated join or invite to a room + // whose version the other side does not support. + ErrIncompatibleRoomVersion = errors.New("incompatible room version") + + // ErrRemote is another server failing to do what was asked of it: + // unreachable, refusing, or answering with something unusable. The + // wrapped message says which server and why. + ErrRemote = errors.New("remote server failed") + + // ErrNotInRoom is a federation request about a room this server is + // not in, or from a server that is not in it. + ErrNotInRoom = errors.New("not in room") + + // ErrBadEvent is an event from another server that is malformed or + // does not verify. + ErrBadEvent = errors.New("bad event") ) // Profiles supplies the display name and avatar that a membership event @@ -92,6 +109,19 @@ type Notification struct { // StateChanged reports whether the room's current state moved. StateChanged bool + + // Origin is the server the event was received from, or empty for an + // event created here. + Origin string + + // Relay asks for an event received from Origin to be passed on to the + // room's other servers. Only a join, leave or knock handed to this + // server through send_join, send_leave or send_knock is relayed: the + // server that sent it is not yet (or no longer) in the room, so it + // relies on the resident server to tell everybody else. An event + // received over /send is never relayed, because its origin sends it to + // every server in the room itself. + Relay bool } // Notifier is told about every event this server persists, in the order @@ -129,12 +159,21 @@ type API struct { // ask for one. defaultRoomVersion roomversion.Version + // remote is the federation client and keys verifies other servers' + // signatures. Both are nil in a build with no federation, which then + // behaves as one with the switch permanently off. + remote RemoteServers + keys auth.Verifier + // locks serialises the writers of each room. Two events racing into one // room would each pick up the same forward extremities and both become // one, forking the room for no reason — and then needing state // resolution to put it back together. Taking a lock per room instead // keeps every locally created event a child of the one before it. locks sync.Map // room ID -> *sync.Mutex + + // backfills serialises the backfills of each room; see backfillLock. + backfills sync.Map // room ID -> *sync.Mutex } // Options configure the component. @@ -164,6 +203,13 @@ type Options struct { // Notifier receives every persisted event. Nil skips the emit stage. Notifier Notifier + // Remote is the federation client: joins, leaves, invites and the + // fetches that fill in a room's history. Nil means no federation. + Remote RemoteServers + + // Keys verifies other servers' signatures. Nil means no federation. + Keys auth.Verifier + Logger *slog.Logger // Now is the clock, so that tests can stamp events deterministically. @@ -211,6 +257,8 @@ func New(opts Options) (*API, error) { now: opts.Now, federationEnabled: opts.FederationEnabled, defaultRoomVersion: version, + remote: opts.Remote, + keys: opts.Keys, }, nil } diff --git a/server/internal/roomserver/roomsync.go b/server/internal/roomserver/roomsync.go index ad92622..9c35ee0 100644 --- a/server/internal/roomserver/roomsync.go +++ b/server/internal/roomserver/roomsync.go @@ -175,13 +175,16 @@ func (r *API) syncState(ctx context.Context, rules roomversion.Rules, visible Vi // timeline events in this window is reported as of the top of the sync // instead: the state may still have been read for the first time. func statePosition(timeline Timeline, to int64, visible VisibleRange) (int64, bool) { + // Zero is the one position nothing can be at, so it stands for "before + // the room began"; a negative Start is inside backfilled history and + // has state like any other position. if len(timeline.Events) > 0 { - return timeline.Start, timeline.Start > 0 + return timeline.Start, timeline.Start != 0 } if visible.To != 0 && (to == 0 || to > visible.To) { to = visible.To } - return to, to > 0 + return to, to != 0 } // shapeState applies the state filter and lazy loading to a state set. diff --git a/server/internal/roomserver/stream.go b/server/internal/roomserver/stream.go index b5bc5bc..56dd4a8 100644 --- a/server/internal/roomserver/stream.go +++ b/server/internal/roomserver/stream.go @@ -36,7 +36,8 @@ type MembershipAt struct { // TimelineRequest is one window of a room's event stream. // // The bounds are the store's: Low exclusive, High inclusive, both -// directions, and zero meaning unbounded at that end. Direction decides +// directions, and zero meaning unbounded at that end. Either can be +// negative, inside history backfilled from other servers. Direction decides // only which end the limit trims and the order the store reads in — the // events always come back oldest first. type TimelineRequest struct { @@ -59,6 +60,13 @@ type TimelineRequest struct { // after the limit is what keeps a filtered page from coming back short // and looking like the end of the room. Filter storage.TimelineFilter + + // Backfill allows a backwards read that runs out of history at the + // bottom of what this server holds to ask the servers in the room for + // more before it answers (see backfill.go). /messages sets it; /sync + // and /context do not, because neither is a client asking to scroll + // back, and neither should wait on another server. + Backfill bool } // Timeline is a window of a room's event stream. @@ -121,7 +129,35 @@ func (r *API) Timeline(ctx context.Context, req TimelineRequest) (Timeline, erro if err != nil { return Timeline{}, err } - return r.timeline(ctx, rules, visible, req) + page, err := r.timeline(ctx, rules, visible, req) + if err != nil { + return Timeline{}, err + } + + // A page that came back short at the bottom of the room is either the + // beginning of the room or the beginning of what this server holds of + // it. Telling a client it has reached the end — no `end` token — when + // it is the second would stop it scrolling for good, so the servers in + // the room are asked first. + for range backfillRounds { + if !r.wantsBackfill(req, visible, page) { + break + } + added, err := r.backfill(ctx, room, rules, req.Limit) + if err != nil { + // The page this server has is still an answer. The client + // will try again the next time it scrolls up. + r.log.Info("could not backfill a room", "room_id", room.ID, "error", err) + break + } + if added == 0 { + break + } + if page, err = r.timeline(ctx, rules, visible, req); err != nil { + return Timeline{}, err + } + } + return page, nil } // timeline is Timeline with the room's rules and the user's visible range @@ -130,7 +166,7 @@ func (r *API) timeline(ctx context.Context, rules roomversion.Rules, visible Vis // An empty window is reported at the bound the caller came in on, so // that the tokens it hands back are still usable. empty := Timeline{Start: req.Low, End: req.Low} - if req.Direction == storage.DirectionBackward && req.High > 0 { + if req.Direction == storage.DirectionBackward && req.High != 0 { empty = Timeline{Start: req.High, End: req.High} } if req.Limit <= 0 { @@ -181,8 +217,10 @@ func (r *API) timeline(ctx context.Context, rules roomversion.Rules, visible Vis } // Start sits below the oldest event, so that handing it back as a High // bound returns the page before this one; End is at the newest, so that - // handing it back as a Low bound returns the page after. - out.Start = stored[0].StreamOrdering - 1 + // handing it back as a Low bound returns the page after. Below the + // event at 1 is -1, not 0: zero would read as "unbounded" and hand the + // client the whole room again rather than the history beneath it. + out.Start = storage.PositionBefore(stored[0].StreamOrdering) out.End = stored[len(stored)-1].StreamOrdering return out, nil } @@ -359,14 +397,62 @@ var strippedStateTypes = []string{ // The member events are what makes an invite legible: a client showing // "Alice invited you" needs Alice's display name, and the invitee's own // member event carries the invite's reason. +// +// For a room this server is not in — an invite that came over federation — +// the room's state is not here, and what is returned is the stripped state +// the inviting server sent along, plus the invite itself. func (r *API) StrippedState(ctx context.Context, roomID string, members []string) ([]StrippedEvent, error) { _, rules, err := r.roomAndRules(ctx, roomID) if err != nil { return nil, err } + if _, err := r.store.CurrentStateEvent(ctx, roomID, event.TypeCreate, ""); isNotFound(err) { + return r.outOfBandStrippedState(ctx, roomID, rules, members) + } else if err != nil { + return nil, err + } return r.strippedState(ctx, roomID, rules, members) } +// outOfBandStrippedState is StrippedState for a room known only from an +// invite. +func (r *API) outOfBandStrippedState(ctx context.Context, roomID string, rules roomversion.Rules, members []string) ([]StrippedEvent, error) { + var out []StrippedEvent + for _, member := range members { + raw, err := r.store.InviteState(ctx, roomID, member) + if err != nil { + return nil, err + } + if len(raw) > 0 { + var stripped []StrippedEvent + if err := json.Unmarshal(raw, &stripped); err != nil { + // What another server sent is not ours to fail a sync over. + r.log.Warn("stored invite state is unreadable", "room_id", roomID, "user_id", member, "error", err) + } + for _, s := range stripped { + if s.Type == event.TypeMember && s.StateKey == member { + continue // the invite itself is added below + } + out = append(out, s) + } + } + } + for _, member := range members { + stored, err := r.store.CurrentStateEvent(ctx, roomID, event.TypeMember, member) + if isNotFound(err) { + continue + } else if err != nil { + return nil, err + } + e, err := parseEvent(stored, rules) + if err != nil { + return nil, err + } + out = append(out, StrippedEvent{Type: e.Type(), StateKey: member, Sender: e.Sender(), Content: e.Content()}) + } + return out, nil +} + // strippedState is StrippedState with the rules in hand. func (r *API) strippedState(ctx context.Context, roomID string, rules roomversion.Rules, members []string) ([]StrippedEvent, error) { var out []StrippedEvent diff --git a/server/internal/roomserver/verify.go b/server/internal/roomserver/verify.go index 345ae7f..3ce34f9 100644 --- a/server/internal/roomserver/verify.go +++ b/server/internal/roomserver/verify.go @@ -4,37 +4,73 @@ import ( "context" "fmt" + "github.com/dariuszjaniak/warp/server/internal/matrix/auth" "github.com/dariuszjaniak/warp/server/internal/matrix/event" "github.com/dariuszjaniak/warp/server/internal/matrix/signing" ) -// localVerifier checks an event's signature against this server's own key. +// eventVerifier checks an event's signature from a named server. // -// One authorisation rule needs somebody else's signature verified: a join -// under the restricted join rule is authorised by a third party named in -// content.join_authorised_via_users_server, and the event must carry a -// signature from *that user's* server. While federation is off the only -// such server that can appear is this one, and its key is already in hand — -// so the rule can be applied in full, locally, without a key fetcher. -// -// Phase 5 replaces this with a verifier that can fetch and cache another -// server's keys. Until then a join authorised via a remote user is refused -// rather than waved through, which is the same answer the federation switch +// This server's own signatures are checked against the key in hand, which +// is what keeps every rule that needs a signature — a restricted join +// authorised by a local user, chiefly — working with federation off. +// Anybody else's go to the key ring, and only while federation is on: with +// it off there is no way to fetch a key, and a signature that cannot be +// checked is refused rather than waved through, the same answer the switch // gives everywhere else. -type localVerifier struct { +type eventVerifier struct { serverName string key *signing.KeyPair + remote auth.Verifier + enabled func() bool } // verifier returns the Verifier the authorisation rules are given. -func (r *API) verifier() *localVerifier { - return &localVerifier{serverName: r.serverName, key: r.signingKey} +func (r *API) verifier() *eventVerifier { + return &eventVerifier{ + serverName: r.serverName, + key: r.signingKey, + remote: r.keys, + enabled: r.federationEnabled, + } } // VerifyEventSignature implements auth.Verifier. -func (v *localVerifier) VerifyEventSignature(_ context.Context, serverName string, e *event.Event) error { - if serverName != v.serverName { +func (v *eventVerifier) VerifyEventSignature(ctx context.Context, serverName string, e *event.Event) error { + if serverName == v.serverName { + return e.VerifySignature(v.serverName, v.key.KeyID(), v.key.Public) + } + if v.remote == nil || !v.enabled() { return fmt.Errorf("%w: cannot verify a signature from %s", ErrFederationDisabled, serverName) } - return e.VerifySignature(v.serverName, v.key.KeyID(), v.key.Public) + return v.remote.VerifyEventSignature(ctx, serverName, e) +} + +// verifyIncoming checks an event from another server before anything else +// looks at it: it must be signed by its sender's server, and its content +// must match its hash. +// +// A content hash mismatch is not a rejection. The signature covers only +// what a redaction keeps, so an event whose signature verifies and whose +// hash does not is an event somebody has redacted — legitimately, or by +// tampering with the parts the signature does not cover — and the spec's +// answer to both is the same: keep the redacted form. +func (r *API) verifyIncoming(ctx context.Context, e *event.Event) (*event.Event, error) { + sender, err := e.SenderServer() + if err != nil { + return nil, fmt.Errorf("%w: %s: %v", ErrBadEvent, e.EventID(), err) + } + if err := r.verifier().VerifyEventSignature(ctx, sender, e); err != nil { + return nil, fmt.Errorf("%w: %s is not signed by %s: %v", ErrBadEvent, e.EventID(), sender, err) + } + if err := e.VerifyContentHash(); err != nil { + redacted, rerr := e.Redact() + if rerr != nil { + return nil, fmt.Errorf("%w: %s: %v", ErrBadEvent, e.EventID(), rerr) + } + r.log.Info("event failed its content hash; using its redacted form", + "event_id", e.EventID(), "sender", e.Sender()) + return redacted, nil + } + return e, nil } diff --git a/server/internal/roomserver/visibility.go b/server/internal/roomserver/visibility.go index 0e9d0dd..b0a0926 100644 --- a/server/internal/roomserver/visibility.go +++ b/server/internal/roomserver/visibility.go @@ -6,6 +6,7 @@ import ( "github.com/dariuszjaniak/warp/server/internal/matrix/event" "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/storage" ) // VisibleRange is the span of a room's event stream a user may read. @@ -22,7 +23,8 @@ type VisibleRange struct { Allowed bool // From is the first position the user may see, exclusive. Zero means - // the start of the room. + // the start of the room, backfilled history included: positions can be + // negative, so zero is the one value that cannot be a real floor. From int64 // To is the last position the user may see, inclusive. Zero means the @@ -43,7 +45,7 @@ func (v VisibleRange) Contains(pos int64) bool { switch { case !v.Allowed: return false - case pos <= v.From: + case v.From != 0 && pos <= v.From: return false case v.To != 0 && pos > v.To: return false @@ -61,11 +63,16 @@ func (v VisibleRange) Clamp(from, to int64) (int64, int64, bool) { if !v.Allowed { return 0, 0, false } - from = max(from, v.From) + // Zero is unbounded at either end, and every other value — negative + // ones inside backfilled history among them — is a real bound, so the + // narrower of two bounds is not simply the larger or smaller number. + if v.From != 0 && (from == 0 || from < v.From) { + from = v.From + } if v.To != 0 && (to == 0 || to > v.To) { to = v.To } - if to != 0 && from >= to { + if from != 0 && to != 0 && from >= to { return 0, 0, false } return from, to, true @@ -207,7 +214,7 @@ func spanStart(changes []membershipChange, historyVisibility string) int64 { } // Exclusive, so one below the event itself: the membership event is the // first thing the user sees, which is how a client renders "you joined". - return changes[start].position - 1 + return storage.PositionBefore(changes[start].position) } // requireReadable refuses a caller who may see nothing of a room, and diff --git a/server/internal/setup/federation.go b/server/internal/setup/federation.go new file mode 100644 index 0000000..1226f26 --- /dev/null +++ b/server/internal/setup/federation.go @@ -0,0 +1,77 @@ +package setup + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "os" + "sync" + + "github.com/dariuszjaniak/warp/server/internal/federationapi" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// fanout is the roomserver's emit stage when more than one thing listens: +// the sync notifier, which wakes long polls, and the federation queue, +// which sends events to other servers. +// +// Each listener is held to the roomserver's rule for a Notifier — never +// block — so calling them in turn under the room lock is fine. +type fanout struct { + mu sync.RWMutex + listeners []roomserver.Notifier +} + +func (f *fanout) add(n roomserver.Notifier) { + f.mu.Lock() + defer f.mu.Unlock() + f.listeners = append(f.listeners, n) +} + +// EventPersisted implements roomserver.Notifier. +func (f *fanout) EventPersisted(ctx context.Context, n roomserver.Notification) { + f.mu.RLock() + defer f.mu.RUnlock() + for _, l := range f.listeners { + l.EventPersisted(ctx, n) + } +} + +// federationStatus adapts the module to the admin API. +type federationStatus struct { + module *federationapi.Module +} + +func (s federationStatus) Running() bool { return s.module.Running() } + +func (s federationStatus) Destinations(ctx context.Context) ([]storage.DestinationSummary, error) { + return s.module.Queue().Destinations(ctx) +} + +func (s federationStatus) ResetBackoff(ctx context.Context, serverName string) error { + return s.module.Queue().ResetBackoff(ctx, serverName) +} + +// federationTLS builds the client TLS configuration for outbound +// federation: the system roots, plus a configured CA bundle. +func federationTLS(caFile string) (*tls.Config, error) { + cfg := &tls.Config{MinVersion: tls.VersionTLS12} + if caFile == "" { + return cfg, nil + } + pool, err := x509.SystemCertPool() + if err != nil || pool == nil { + pool = x509.NewCertPool() + } + pem, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("read federation.tls_ca_file: %w", err) + } + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("federation.tls_ca_file %s contains no PEM certificates", caFile) + } + cfg.RootCAs = pool + return cfg, nil +} diff --git a/server/internal/setup/phase5_test.go b/server/internal/setup/phase5_test.go new file mode 100644 index 0000000..f361b9b --- /dev/null +++ b/server/internal/setup/phase5_test.go @@ -0,0 +1,441 @@ +package setup_test + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "log/slog" + "math/big" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/dariuszjaniak/warp/server/internal/config" + "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/matrix/signing" + "github.com/dariuszjaniak/warp/server/internal/setup" + "github.com/dariuszjaniak/warp/server/internal/userapi" +) + +// The end-to-end federation tests: complete servers, each behind its own +// TLS listener, finding each other through a resolver that maps their +// names to those listeners. Everything between the client API on one +// server and the client API on the other is real — X-Matrix signing, key +// fetches, the routes, the gate, the outbound queue. + +// fedNode is one server of an end-to-end federation. +type fedNode struct { + name string + srv *setup.Server + ts *httptest.Server + + // handler is the server's handler, set once it exists. The listener + // has to be up before the server is built — every server needs every + // other's address — so it starts out answering 503. + handler atomic.Pointer[http.Handler] +} + +func (n *fedNode) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h := n.handler.Load() + if h == nil { + http.Error(w, "not ready", http.StatusServiceUnavailable) + return + } + (*h).ServeHTTP(w, r) +} + +// newFederation starts servers that can reach each other. +func newFederation(t *testing.T, names ...string) map[string]*fedNode { + t.Helper() + nodes := map[string]*fedNode{} + for _, name := range names { + node := &fedNode{name: name} + node.ts = httptest.NewUnstartedServer(node) + node.ts.StartTLS() + t.Cleanup(node.ts.Close) + nodes[name] = node + } + + // Every test listener has the same certificate, for "example.com"; the + // Host is that name with the listener's port, and the address is the + // listener. That is the delegated-by-SRV shape: TLS is checked against + // one name and the connection goes somewhere else. + resolve := func(_ context.Context, name string) (federation.Resolved, error) { + node, ok := nodes[name] + if !ok { + return federation.Resolved{}, &net.DNSError{Err: "no such host", Name: name, IsNotFound: true} + } + addr := node.ts.Listener.Addr().String() + _, port, _ := net.SplitHostPort(addr) + return federation.Resolved{Addresses: []string{addr}, Host: "example.com:" + port}, nil + } + anyTS := nodes[names[0]].ts + clientTLS := anyTS.Client().Transport.(*http.Transport).TLSClientConfig.Clone() + + for _, name := range names { + node := nodes[name] + key, err := signing.Generate("") + if err != nil { + t.Fatal(err) + } + keyPath := filepath.Join(t.TempDir(), name+".key") + if err := key.Save(keyPath); err != nil { + t.Fatal(err) + } + cfg := config.Default() + cfg.Global.ServerName = name + cfg.Global.PrivateKeyPath = keyPath + cfg.Database.ConnectionString = "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), name+".db")) + + "?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)" + cfg.Registration.Enabled = true + cfg.ClientAPI.RateLimit.Enabled = false + cfg.Federation.Enabled = true + if err := cfg.Validate(); err != nil { + t.Fatalf("config: %v", err) + } + srv, err := setup.New(t.Context(), cfg, setup.Options{ + Logger: slog.New(slog.DiscardHandler), + FederationResolve: resolve, + FederationTLS: clientTLS, + }) + if err != nil { + t.Fatalf("setup.New(%s): %v", name, err) + } + t.Cleanup(func() { _ = srv.Close() }) + node.srv = srv + h := srv.Handler() + node.handler.Store(&h) + t.Cleanup(srv.FederationModule().Stop) + if err := srv.FederationModule().Start(t.Context()); err != nil { + t.Fatalf("start federation on %s: %v", name, err) + } + } + return nodes +} + +// eventually polls until check passes or the deadline is reached. +func eventually(t *testing.T, what string, check func() bool) { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if check() { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +// messageBodies reads the message bodies of a room's recent history. +func messageBodies(t *testing.T, srv *setup.Server, roomID, token string) []string { + t.Helper() + rec := getWithToken(t, srv, "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/messages?dir=b&limit=50", token) + if rec.Code != http.StatusOK { + return nil + } + var resp struct { + Chunk []struct { + Type string `json:"type"` + Content struct { + Body string `json:"body"` + } `json:"content"` + } `json:"chunk"` + } + _ = json.Unmarshal(rec.Body.Bytes(), &resp) + var out []string + for _, e := range resp.Chunk { + if e.Type == "m.room.message" { + out = append(out, e.Content.Body) + } + } + return out +} + +func contains(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + +func sendMessage(t *testing.T, srv *setup.Server, roomID, token, txn, body string) { + t.Helper() + rec := put(t, srv, "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/send/m.room.message/"+txn, + map[string]any{"msgtype": "m.text", "body": body}, token) + if rec.Code != http.StatusOK { + t.Fatalf("send %q: status %d: %s", body, rec.Code, rec.Body.String()) + } +} + +func TestFederationEndToEnd(t *testing.T) { + if testing.Short() { + t.Skip("starts two servers") + } + nodes := newFederation(t, "hs-a.example", "hs-b.example") + a, b := nodes["hs-a.example"].srv, nodes["hs-b.example"].srv + + aliceToken := registerThrough(t, a, "alice") + bobToken := registerThrough(t, b, "bob") + + rec := post(t, a, "/_matrix/client/v3/createRoom", map[string]any{ + "preset": "public_chat", "name": "Across", "room_alias_name": "across", + }, aliceToken) + if rec.Code != http.StatusOK { + t.Fatalf("createRoom: %d %s", rec.Code, rec.Body.String()) + } + roomID, _ := decode[map[string]any](t, rec)["room_id"].(string) + + // Bob joins by alias: the alias is resolved by asking A, then the join + // goes through A. + rec = post(t, b, "/_matrix/client/v3/join/"+url.PathEscape("#across:hs-a.example"), map[string]any{}, bobToken) + if rec.Code != http.StatusOK { + t.Fatalf("remote join: %d %s", rec.Code, rec.Body.String()) + } + if got, _ := decode[map[string]any](t, rec)["room_id"].(string); got != roomID { + t.Fatalf("joined %q, want %q", got, roomID) + } + + // Alice hears of the join without any queue involved: it went + // through her server. + rec = getWithToken(t, a, "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/joined_members", aliceToken) + if !strings.Contains(rec.Body.String(), "@bob:hs-b.example") { + t.Fatalf("A's joined members lack bob: %s", rec.Body.String()) + } + + // Messages in both directions go through the queues. + sendMessage(t, a, roomID, aliceToken, "a1", "hello from A") + eventually(t, "A's message on B", func() bool { + return contains(messageBodies(t, b, roomID, bobToken), "hello from A") + }) + sendMessage(t, b, roomID, bobToken, "b1", "hello from B") + eventually(t, "B's message on A", func() bool { + return contains(messageBodies(t, a, roomID, aliceToken), "hello from B") + }) + + // Remote profile lookup. + rec = getWithToken(t, b, "/_matrix/client/v3/profile/@alice:hs-a.example", bobToken) + if rec.Code != http.StatusOK { + t.Errorf("remote profile: %d %s", rec.Code, rec.Body.String()) + } + + // Switching A's federation off: B's next message cannot be delivered, + // and B records the failure. Switching it back on and clearing the + // backoff delivers it. + if err := a.Settings().SetFederationEnabled(t.Context(), false); err != nil { + t.Fatal(err) + } + a.FederationModule().Stop() + sendMessage(t, b, roomID, bobToken, "b2", "while A is away") + eventually(t, "B to record A as failing", func() bool { + ds, err := b.FederationModule().Queue().Destinations(t.Context()) + if err != nil { + return false + } + for _, d := range ds { + if d.ServerName == "hs-a.example" && d.Failures > 0 && d.QueuedPDUs > 0 { + return true + } + } + return false + }) + if contains(messageBodies(t, a, roomID, aliceToken), "while A is away") { + t.Fatal("a message reached A while its federation was off") + } + + // The admin API shows the backlog and the backoff. + root, err := b.Users().CreateAccount(t.Context(), userapi.NewAccountRequest{ + Localpart: "root", Password: "sUp3rs3kr1t", Admin: true, + }) + if err != nil { + t.Fatal(err) + } + session, err := b.Users().CreateSession(t.Context(), userapi.SessionRequest{Localpart: root.Localpart}) + if err != nil { + t.Fatal(err) + } + rec = getWithToken(t, b, "/_warp/admin/v1/federation/destinations", session.AccessToken) + if rec.Code != http.StatusOK { + t.Fatalf("GET destinations: %d %s", rec.Code, rec.Body.String()) + } + var listing struct { + Running bool `json:"running"` + Destinations []struct { + ServerName string `json:"server_name"` + Failures int `json:"failures"` + QueuedPDUs int `json:"queued_pdus"` + } `json:"destinations"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &listing); err != nil { + t.Fatal(err) + } + if !listing.Running || len(listing.Destinations) == 0 || + listing.Destinations[0].ServerName != "hs-a.example" || listing.Destinations[0].Failures == 0 { + t.Errorf("destinations = %s", rec.Body.String()) + } + + if err := a.Settings().SetFederationEnabled(t.Context(), true); err != nil { + t.Fatal(err) + } + if err := a.FederationModule().Start(t.Context()); err != nil { + t.Fatal(err) + } + if err := b.FederationModule().Queue().ResetBackoff(t.Context(), "hs-a.example"); err != nil { + t.Fatal(err) + } + eventually(t, "the held message on A", func() bool { + return contains(messageBodies(t, a, roomID, aliceToken), "while A is away") + }) +} + +func TestFederationEndpointsAreGated(t *testing.T) { + srv := newSigningServer(t, nil) + for _, path := range []string{ + "/_matrix/key/v2/server", + "/_matrix/federation/v1/version", + } { + if rec := get(t, srv, http.MethodGet, path); rec.Code != http.StatusNotFound { + t.Errorf("%s with federation off = %d, want 404", path, rec.Code) + } + } + + if err := srv.Settings().SetFederationEnabled(t.Context(), true); err != nil { + t.Fatal(err) + } + rec := get(t, srv, http.MethodGet, "/_matrix/key/v2/server") + if rec.Code != http.StatusOK { + t.Fatalf("key server with federation on = %d", rec.Code) + } + if name, _ := decode[map[string]any](t, rec)["server_name"].(string); name != "warp.example" { + t.Errorf("key response server_name = %q", name) + } + + // An unsigned request to an authenticated endpoint is refused. + rec = get(t, srv, http.MethodGet, "/_matrix/federation/v1/event/$abc") + if rec.Code != http.StatusUnauthorized { + t.Errorf("unsigned /event = %d, want 401", rec.Code) + } +} + +// writeSelfSignedCert writes a certificate for 127.0.0.1 and its key, and +// returns their paths and a pool that trusts the certificate. +func writeSelfSignedCert(t *testing.T) (string, string, *x509.CertPool) { + t.Helper() + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "warp.example"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &priv.PublicKey, priv) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalECPrivateKey(priv) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + certPath, keyPath := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem") + if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600); err != nil { + t.Fatal(err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + pool.AddCert(cert) + return certPath, keyPath, pool +} + +func TestFederationListenerServesTLS(t *testing.T) { + certPath, keyPath, pool := writeSelfSignedCert(t) + srv := newSigningServer(t, func(c *config.Config) { + c.Listen.Address = "127.0.0.1:0" + c.Listen.FederationAddress = "127.0.0.1:0" + c.Listen.TLSCertFile = certPath + c.Listen.TLSKeyFile = keyPath + c.Federation.Enabled = true + c.Database.ConnectionString = "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "warp.db")) + }) + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- srv.Run(ctx) }() + defer func() { + cancel() + <-done + }() + + var addr string + for range 200 { + if addr = srv.FederationAddr(); addr != "" { + break + } + time.Sleep(10 * time.Millisecond) + } + if addr == "" { + t.Fatal("the federation listener never started") + } + if !srv.FederationModule().Running() { + t.Error("Run did not start the federation module with federation on") + } + + client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}} + resp, err := client.Get("https://" + addr + "/_matrix/key/v2/server") + if err != nil { + t.Fatalf("GET over TLS: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("key server over TLS = %d", resp.StatusCode) + } + + // The runtime switch stops the module while Run is serving. + if err := srv.Settings().SetFederationEnabled(t.Context(), false); err != nil { + t.Fatal(err) + } + if srv.FederationModule().Running() { + t.Error("switching federation off did not stop the module") + } + if err := srv.Settings().SetFederationEnabled(t.Context(), true); err != nil { + t.Fatal(err) + } + if !srv.FederationModule().Running() { + t.Error("switching federation back on did not start the module") + } +} + +func TestFederationListenerNeedsACertificate(t *testing.T) { + cfg := config.Default() + cfg.Global.ServerName = "warp.example" + cfg.Listen.FederationAddress = ":8448" + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "tls_cert_file") { + t.Errorf("Validate = %v, want a complaint about the certificate", err) + } +} diff --git a/server/internal/setup/routes.go b/server/internal/setup/routes.go index b93c821..e43fd8c 100644 --- a/server/internal/setup/routes.go +++ b/server/internal/setup/routes.go @@ -14,10 +14,10 @@ import ( // actually uses from it work. Accounts, sessions, devices, profiles, // account data, rooms and — with Phase 4 — the event stream all do, which // is enough for a client to be used. What is still missing is end-to-end -// encryption (Phase 7), media (Phase 6), push (Phase 8) and federation -// (Phase 5); a client discovers each of those from the endpoint it needs -// rather than from this list, which is why their absence does not make a -// version listed here a lie. +// encryption (Phase 7), media (Phase 6) and push (Phase 8); a client +// discovers each of those from the endpoint it needs rather than from this +// list, which is why their absence does not make a version listed here a +// lie. var SpecVersions = []string{"v1.13", "v1.14"} // UnstableFeatures advertises MSC support. Simplified Sliding Sync @@ -62,6 +62,9 @@ func (s *Server) routes() *httputil.Router { r.Handle("GET /.well-known/matrix/server", s.federation.ProtectFunc(s.handleWellKnownServer)) s.clientAPI.RegisterRoutes(r) + if s.fedAPI != nil { + s.fedAPI.RegisterRoutes(r) + } s.adminAPI.RegisterRoutes(r) r.HandleFunc("GET /healthz", s.handleHealth) diff --git a/server/internal/setup/server.go b/server/internal/setup/server.go index dc65794..5f7bf72 100644 --- a/server/internal/setup/server.go +++ b/server/internal/setup/server.go @@ -5,6 +5,7 @@ package setup import ( "context" + "crypto/tls" "errors" "fmt" "log/slog" @@ -18,6 +19,7 @@ import ( "github.com/dariuszjaniak/warp/server/internal/clientapi" "github.com/dariuszjaniak/warp/server/internal/config" "github.com/dariuszjaniak/warp/server/internal/federation" + "github.com/dariuszjaniak/warp/server/internal/federationapi" "github.com/dariuszjaniak/warp/server/internal/httputil" "github.com/dariuszjaniak/warp/server/internal/matrix/signing" "github.com/dariuszjaniak/warp/server/internal/metrics" @@ -46,11 +48,30 @@ type Server struct { clientAPI *clientapi.ClientAPI adminAPI *adminapi.AdminAPI + // The federation components. All nil when the server started without + // a signing key, because nothing that federates can work without one. + fedClient *federation.Client + keyRing *federation.KeyRing + fedModule *federationapi.Module + fedAPI *federationapi.Server + + // tlsConfig serves the federation listener; nil when there is none. + tlsConfig *tls.Config + + // running is set while Run is serving. The federation switch only + // starts and stops the module while it is: a server that was built and + // never run must not leave a queue running behind it. + running atomic.Bool + handler http.Handler // addr is the address Run is listening on; empty until then. addr atomic.Pointer[string] + // fedAddr is the TLS federation listener's address while Run is + // serving one; empty otherwise. + fedAddr atomic.Pointer[string] + // ownsDB is false when the caller supplied the database and will close // it itself. ownsDB bool @@ -69,6 +90,13 @@ type Options struct { // Only tests and `warp migrate` use it: a real server must be able to // sign. SkipSigningKey bool + + // FederationResolve replaces server discovery, and FederationTLS the + // outbound TLS configuration. Tests that run several servers in one + // process use them to point server names at local listeners whose + // certificates they trust. + FederationResolve federation.ResolveFunc + FederationTLS *tls.Config } // New builds the server: storage, migrations, settings, routes. It does not @@ -128,12 +156,24 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) settingsService.FederationEnabled, federation.NewPolicy(cfg.Federation.Allowlist, cfg.Federation.Denylist), ) - // Phase 5 replaces this with starting and stopping the federation - // module; for now it is the audit trail for a runtime toggle. + // The switch starts and stops the federation module. The gate and the + // client read the switch on every request and need no telling. settingsService.Subscribe(func(snap settings.Snapshot) { log.Info("federation switch applied", "enabled", snap.FederationEnabled) + if s.fedModule != nil && s.running.Load() { + s.fedModule.Apply(context.Background(), snap.FederationEnabled) + } }) + if cfg.Listen.FederationAddress != "" { + cert, err := tls.LoadX509KeyPair(cfg.Listen.TLSCertFile, cfg.Listen.TLSKeyFile) + if err != nil { + _ = s.closeDB() + return nil, fmt.Errorf("load the federation listener's certificate: %w", err) + } + s.tlsConfig = &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12} + } + s.users, err = userapi.New(userapi.Options{ ServerName: cfg.Global.ServerName, Store: s.db.Users(), @@ -159,8 +199,40 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) // The notifier is built first and handed to both: it is the // roomserver's emit stage and the syncapi's wake-up, which is exactly // the seam Phase 3 left for this. + // + // Federation hangs off the same key. The client and the key ring are + // built whether or not the switch is on — they refuse everything while + // it is off — and the outbound queue is a second listener on the + // roomserver's emit stage next to the notifier. if s.signingKey != nil { notifier := syncapi.NewNotifier() + emit := &fanout{} + emit.add(notifier) + + clientTLS := opts.FederationTLS + if clientTLS == nil { + if clientTLS, err = federationTLS(cfg.Federation.TLSCAFile); err != nil { + _ = s.closeDB() + return nil, err + } + } + s.fedClient = federation.NewClient(federation.ClientOptions{ + ServerName: cfg.Global.ServerName, + Key: s.signingKey, + Resolve: opts.FederationResolve, + TLSConfig: clientTLS, + Enabled: settingsService.FederationEnabled, + Policy: s.federation.Policy(), + }) + s.keyRing = federation.NewKeyRing(federation.KeyRingOptions{ + ServerName: cfg.Global.ServerName, + Key: s.signingKey, + Store: s.db.Federation(), + Client: s.fedClient, + Notaries: cfg.Federation.TrustedKeyServers, + Validity: cfg.Federation.KeyValidityPeriod, + Logger: log, + }) s.rooms, err = roomserver.New(roomserver.Options{ ServerName: cfg.Global.ServerName, @@ -168,7 +240,9 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) Profiles: s.users, SigningKey: s.signingKey, FederationEnabled: settingsService.FederationEnabled, - Notifier: notifier, + Notifier: emit, + Remote: s.fedClient, + Keys: s.keyRing, Logger: log, }) if err != nil { @@ -176,6 +250,17 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) return nil, err } + queue := federationapi.NewQueue(federationapi.QueueOptions{ + ServerName: cfg.Global.ServerName, + Store: s.db.Federation(), + Sender: s.fedClient, + Rooms: s.rooms, + Policy: s.federation.Policy(), + Logger: log, + }) + emit.add(queue) + s.fedModule = federationapi.NewModule(queue, log) + s.sync, err = syncapi.New(syncapi.Options{ ServerName: cfg.Global.ServerName, Rooms: s.rooms, @@ -191,10 +276,36 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) _ = s.closeDB() return nil, err } + s.sync.SetOutbound(federationapi.NewEDUs(queue, s.rooms)) + + s.fedAPI, err = federationapi.NewServer(federationapi.ServerOptions{ + ServerName: cfg.Global.ServerName, + Gate: s.federation, + Keys: s.keyRing, + Rooms: s.rooms, + Ephemeral: s.sync, + Profiles: s.users, + Store: s.db.Federation(), + Logger: log, + }) + if err != nil { + _ = s.closeDB() + return nil, err + } } else { log.Warn("started without a signing key: rooms and /sync are unavailable") } + // The interfaces below are left nil rather than set to a nil pointer + // when there is no federation: a nil *federation.Client in an + // interface is not a nil interface, and would be called. + var remoteProfiles clientapi.RemoteProfiles + var fedStatus adminapi.FederationStatus + if s.fedClient != nil { + remoteProfiles = s.fedClient + fedStatus = federationStatus{module: s.fedModule} + } + s.clientAPI, err = clientapi.New(clientapi.Options{ ServerName: cfg.Global.ServerName, ClientBaseURL: cfg.ClientBaseURL(), @@ -203,6 +314,7 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) Sync: s.sync, Settings: settingsService, Federation: s.federation, + RemoteProfiles: remoteProfiles, Registration: cfg.Registration, RateLimit: cfg.ClientAPI.RateLimit, TrustXForwardedFor: cfg.ClientAPI.TrustXForwardedFor, @@ -214,13 +326,14 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) } s.adminAPI, err = adminapi.New(adminapi.Options{ - ServerName: cfg.Global.ServerName, - Users: s.users, - Rooms: s.rooms, - Settings: settingsService, - Federation: s.federation, - Database: s.db, - Logger: log, + ServerName: cfg.Global.ServerName, + Users: s.users, + Rooms: s.rooms, + Settings: settingsService, + Federation: s.federation, + FederationStatus: fedStatus, + Database: s.db, + Logger: log, }) if err != nil { _ = s.closeDB() @@ -292,29 +405,65 @@ func (s *Server) SigningKey() *signing.KeyPair { return s.signingKey } // Run listens and serves until ctx is cancelled, then shuts down // gracefully: it stops accepting connections and gives in-flight requests // up to listen.shutdown_timeout to finish. +// +// With listen.federation_address set there are two listeners, the plain +// one and a TLS one for other servers, serving the same handler: what a +// request may reach is decided by the gate and the routes, not by which +// port it came in on. The federation module is started here rather than +// in New, so that a server that is built and never run leaves nothing +// running behind it, and is stopped before the listeners close. func (s *Server) Run(ctx context.Context) error { listener, err := net.Listen("tcp", s.cfg.Listen.Address) if err != nil { return fmt.Errorf("listen on %s: %w", s.cfg.Listen.Address, err) } + listeners := []net.Listener{listener} + if s.tlsConfig != nil { + fedListener, err := net.Listen("tcp", s.cfg.Listen.FederationAddress) + if err != nil { + _ = listener.Close() + return fmt.Errorf("listen on %s: %w", s.cfg.Listen.FederationAddress, err) + } + listeners = append(listeners, tls.NewListener(fedListener, s.tlsConfig)) + } addr := listener.Addr().String() s.addr.Store(&addr) defer s.addr.Store(nil) - httpServer := &http.Server{ - Handler: s.handler, - ReadHeaderTimeout: s.cfg.Listen.ReadHeaderTimeout, - ReadTimeout: s.cfg.Listen.ReadTimeout, - WriteTimeout: s.cfg.Listen.WriteTimeout, - IdleTimeout: s.cfg.Listen.IdleTimeout, - MaxHeaderBytes: s.cfg.Listen.MaxHeaderBytes, - ErrorLog: slog.NewLogLogger(s.log.Handler(), slog.LevelDebug), - BaseContext: func(net.Listener) context.Context { return ctx }, + newHTTPServer := func() *http.Server { + return &http.Server{ + Handler: s.handler, + ReadHeaderTimeout: s.cfg.Listen.ReadHeaderTimeout, + ReadTimeout: s.cfg.Listen.ReadTimeout, + WriteTimeout: s.cfg.Listen.WriteTimeout, + IdleTimeout: s.cfg.Listen.IdleTimeout, + MaxHeaderBytes: s.cfg.Listen.MaxHeaderBytes, + ErrorLog: slog.NewLogLogger(s.log.Handler(), slog.LevelDebug), + BaseContext: func(net.Listener) context.Context { return ctx }, + } } + s.running.Store(true) + defer s.running.Store(false) + if s.fedModule != nil { + if s.settings.FederationEnabled() { + if err := s.fedModule.Start(ctx); err != nil { + s.log.Error("could not start federation", "error", err) + } + } + defer s.fedModule.Stop() + } + + federationAddr := "" + if len(listeners) > 1 { + federationAddr = listeners[1].Addr().String() + s.fedAddr.Store(&federationAddr) + defer s.fedAddr.Store(nil) + } s.log.Info("warp is listening", "address", addr, + "federation_address", federationAddr, "server_name", s.cfg.Global.ServerName, "version", buildinfo.String(), "database", s.db.Driver(), @@ -323,32 +472,52 @@ func (s *Server) Run(ctx context.Context) error { "presence", s.cfg.Sync.Presence, ) - errCh := make(chan error, 1) - go func() { - err := httpServer.Serve(listener) - if errors.Is(err, http.ErrServerClosed) { - err = nil - } - errCh <- err - }() + servers := make([]*http.Server, len(listeners)) + errCh := make(chan error, len(listeners)) + for i, l := range listeners { + servers[i] = newHTTPServer() + go func(srv *http.Server, l net.Listener) { + err := srv.Serve(l) + if errors.Is(err, http.ErrServerClosed) { + err = nil + } + errCh <- err + }(servers[i], l) + } + var serveErr error select { - case err := <-errCh: - return err + case serveErr = <-errCh: + // One listener failing takes the server down: half a homeserver + // is worse than a restart. case <-ctx.Done(): } s.log.Info("shutting down", "timeout", s.cfg.Listen.ShutdownTimeout) + if s.fedModule != nil { + s.fedModule.Stop() + } shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.cfg.Listen.ShutdownTimeout) defer cancel() - if err := httpServer.Shutdown(shutdownCtx); err != nil { - s.log.Warn("graceful shutdown timed out; closing connections", "error", err) - _ = httpServer.Close() + for _, srv := range servers { + if err := srv.Shutdown(shutdownCtx); err != nil { + s.log.Warn("graceful shutdown timed out; closing connections", "error", err) + _ = srv.Close() + } } - // Serve has returned by now; surface anything it reported. - if err := <-errCh; err != nil { - return err + // Every Serve has returned by now; surface anything they reported. + remaining := len(servers) + if serveErr != nil { + remaining-- + } + for range remaining { + if err := <-errCh; err != nil && serveErr == nil { + serveErr = err + } + } + if serveErr != nil { + return serveErr } s.log.Info("stopped") return nil @@ -356,6 +525,9 @@ func (s *Server) Run(ctx context.Context) error { // Close releases the resources New acquired. func (s *Server) Close() error { + if s.fedModule != nil { + s.fedModule.Stop() + } if s.sync != nil { // The typing tracker holds a timer per typing user; a server // shutting down should not wait for them. @@ -378,3 +550,17 @@ func (s *Server) closeDB() error { func contextWithTimeout(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) { return context.WithTimeout(ctx, d) } + +// FederationModule exposes the federation module, or nil when the server +// started without a signing key. Tests that never call Run start it +// themselves. +func (s *Server) FederationModule() *federationapi.Module { return s.fedModule } + +// FederationAddr reports the address of the TLS federation listener while +// Run is serving one, or "". +func (s *Server) FederationAddr() string { + if p := s.fedAddr.Load(); p != nil { + return *p + } + return "" +} diff --git a/server/internal/storage/backfill_test.go b/server/internal/storage/backfill_test.go new file mode 100644 index 0000000..dbbe6ab --- /dev/null +++ b/server/internal/storage/backfill_test.go @@ -0,0 +1,205 @@ +package storage_test + +import ( + "slices" + "testing" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Backfilled history: events numbered downwards from -1, below everything +// the room already holds, and every stream read that has to find them +// there. + +// timelineIDs reads a window and returns its event IDs in the order the +// store gave them. +func timelineIDs(t *testing.T, rooms storage.RoomStore, req storage.TimelineRequest) []string { + t.Helper() + req.RoomID = testRoomID + if req.Limit == 0 { + req.Limit = 100 + } + events, err := rooms.Timeline(t.Context(), req) + if err != nil { + t.Fatalf("Timeline(%+v): %v", req, err) + } + return eventIDsOf(events) +} + +func TestBackfillSortsBelowEverythingAlreadyHeld(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + rooms := migratedRoomDB(t, driver).Rooms() + if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { + t.Fatalf("CreateRoom: %v", err) + } + + // The room as a server that joined it part-way through holds + // it: the state it was handed as outliers, then its own join + // and what followed in the timeline. + create := newTestEvent(testRoomID, "$create", "m.room.create", stateKey("")) + topic := newTestEvent(testRoomID, "$topic", "m.room.topic", stateKey("")) + if err := rooms.WriteOutliers(ctx, testRoomID, []storage.StoredEvent{create, topic}); err != nil { + t.Fatalf("WriteOutliers: %v", err) + } + state := []storage.StateEntry{ + {Type: "m.room.create", StateKey: "", EventID: "$create"}, + {Type: "m.room.topic", StateKey: "", EventID: "$topic"}, + } + if _, err := rooms.Write(ctx, storage.RoomWrite{ + Event: newTestEvent(testRoomID, "$join", "m.room.member", stateKey("@bob:warp.example")), + StateBefore: state, + StateDelta: append(slices.Clone(state), storage.StateEntry{ + Type: "m.room.member", StateKey: "@bob:warp.example", EventID: "$join", Membership: "join", + }), + StateDeltaIsWholeState: true, + Extremities: []string{"$join"}, + }); err != nil { + t.Fatalf("Write the join: %v", err) + } + writeMessages(t, rooms, testRoomID, "$live") + topBefore, err := rooms.MaxStreamOrdering(ctx) + if err != nil { + t.Fatalf("MaxStreamOrdering: %v", err) + } + + // The history before the join, oldest first. The create event + // and the topic are outliers already and are promoted; the + // message is new. + if err := rooms.WriteBackfill(ctx, testRoomID, []storage.BackfillEvent{ + {Event: create}, + {Event: topic, StateBefore: state[:1]}, + {Event: newTestEvent(testRoomID, "$old", "m.room.message", nil), StateBefore: state}, + }); err != nil { + t.Fatalf("WriteBackfill: %v", err) + } + + want := map[string]int64{"$create": -3, "$topic": -2, "$old": -1} + for id, pos := range want { + stored, err := rooms.Event(ctx, testRoomID, id) + if err != nil { + t.Fatalf("Event(%s): %v", id, err) + } + if stored.StreamOrdering != pos || stored.Outlier { + t.Errorf("%s: position %d, outlier %v; want %d in the timeline", + id, stored.StreamOrdering, stored.Outlier, pos) + } + } + + // The whole room, forwards and unbounded, is history first. + wantAll := []string{"$create", "$topic", "$old", "$join", "$live"} + if got := timelineIDs(t, rooms, storage.TimelineRequest{Direction: storage.DirectionForward}); !slices.Equal(got, wantAll) { + t.Errorf("forwards from the start = %v, want %v", got, wantAll) + } + + // Negative bounds, both ends, both directions. + for _, tc := range []struct { + name string + req storage.TimelineRequest + want []string + }{ + { + name: "backwards from below the first live event", + req: storage.TimelineRequest{High: storage.PositionBefore(1), Direction: storage.DirectionBackward}, + want: []string{"$old", "$topic", "$create"}, + }, + { + name: "backwards from a negative position", + req: storage.TimelineRequest{High: -2, Direction: storage.DirectionBackward}, + want: []string{"$topic", "$create"}, + }, + { + name: "forwards past a negative position into the live stream", + req: storage.TimelineRequest{Low: -2, Direction: storage.DirectionForward, Limit: 2}, + want: []string{"$old", "$join"}, + }, + { + name: "a window entirely below zero", + req: storage.TimelineRequest{Low: -3, High: -1, Direction: storage.DirectionForward}, + want: []string{"$topic", "$old"}, + }, + } { + if got := timelineIDs(t, rooms, tc.req); !slices.Equal(got, tc.want) { + t.Errorf("%s: %v, want %v", tc.name, got, tc.want) + } + } + + // History changes neither the top of the stream nor anything + // an incremental sync looks at. + topAfter, err := rooms.MaxStreamOrdering(ctx) + if err != nil { + t.Fatalf("MaxStreamOrdering: %v", err) + } + if topAfter > topBefore { + t.Errorf("the stream top moved from %d to %d", topBefore, topAfter) + } + if changed, err := rooms.RoomsWithEvents(ctx, []string{testRoomID}, topAfter, 0); err != nil { + t.Fatalf("RoomsWithEvents: %v", err) + } else if len(changed) != 0 { + t.Errorf("RoomsWithEvents since the top = %v, want nothing", changed) + } + if changed, err := rooms.RoomsWithEvents(ctx, []string{testRoomID}, 0, -1); err != nil { + t.Fatalf("RoomsWithEvents: %v", err) + } else if len(changed) != 1 { + t.Errorf("RoomsWithEvents over the history = %v, want the room", changed) + } + extremities, err := rooms.Extremities(ctx, testRoomID) + if err != nil { + t.Fatalf("Extremities: %v", err) + } + if len(extremities) != 1 || extremities[0].EventID != "$live" { + t.Errorf("extremities = %v, want just $live", eventIDsOf(extremities)) + } + current, err := rooms.CurrentState(ctx, testRoomID) + if err != nil { + t.Fatalf("CurrentState: %v", err) + } + if len(current) != 3 { + t.Errorf("current state = %+v, want the three tuples from before", current) + } + + // The state at a position inside the history is the snapshot + // recorded against the event there, plus the event. + for _, tc := range []struct { + pos int64 + want int + }{{-4, 0}, {-3, 1}, {-2, 2}, {-1, 2}} { + entries, err := rooms.StateAtStream(ctx, testRoomID, tc.pos) + if err != nil { + t.Fatalf("StateAtStream(%d): %v", tc.pos, err) + } + if len(entries) != tc.want { + t.Errorf("StateAtStream(%d) = %+v, want %d entries", tc.pos, entries, tc.want) + } + } + + // A second batch goes below the first. An event already in the + // timeline is skipped and takes no position. + if err := rooms.WriteBackfill(ctx, testRoomID, []storage.BackfillEvent{ + {Event: newTestEvent(testRoomID, "$older", "m.room.message", nil), StateBefore: state}, + {Event: newTestEvent(testRoomID, "$old", "m.room.message", nil), StateBefore: state}, + }); err != nil { + t.Fatalf("WriteBackfill again: %v", err) + } + older, err := rooms.Event(ctx, testRoomID, "$older") + if err != nil { + t.Fatalf("Event($older): %v", err) + } + if older.StreamOrdering != -4 { + t.Errorf("$older is at %d, want -4", older.StreamOrdering) + } + if old, _ := rooms.Event(ctx, testRoomID, "$old"); old.StreamOrdering != -1 { + t.Errorf("$old moved to %d", old.StreamOrdering) + } + }) + } +} + +func TestPositionBeforeSkipsZero(t *testing.T) { + for pos, want := range map[int64]int64{5: 4, 2: 1, 1: -1, -1: -2, -7: -8} { + if got := storage.PositionBefore(pos); got != want { + t.Errorf("PositionBefore(%d) = %d, want %d", pos, got, want) + } + } +} diff --git a/server/internal/storage/federation.go b/server/internal/storage/federation.go new file mode 100644 index 0000000..64e448f --- /dev/null +++ b/server/internal/storage/federation.go @@ -0,0 +1,123 @@ +package storage + +import "context" + +// ServerKey is one signing key of another server, as its key server or a +// notary reported it. +type ServerKey struct { + ServerName string + + // KeyID is the full identifier, algorithm included: "ed25519:abc". + KeyID string + + // PublicKey is the unpadded base64 key, as it is on the wire. + PublicKey string + + // ValidUntilTS is how long the server vouched for the key, in + // milliseconds since the epoch. + ValidUntilTS int64 + + // ExpiredTS is when the server stopped signing with the key, for one + // it listed as an old_verify_key; zero for a key still in use. + ExpiredTS int64 + + FetchedAtMS int64 +} + +// The kinds of item in the outbound queue. +const ( + QueueKindPDU = "pdu" + QueueKindEDU = "edu" +) + +// QueuedItem is one PDU or EDU waiting for a destination. +type QueuedItem struct { + // ID is assigned by Enqueue and orders a destination's items. + ID int64 + + Destination string + + // Kind is QueueKindPDU or QueueKindEDU. + Kind string + + // RoomID and EventID describe a PDU, for logs and the admin API. + RoomID string + EventID string + + // Payload is the JSON that goes on the wire. + Payload []byte + + QueuedAtMS int64 +} + +// Destination is the delivery health of one remote server. +type Destination struct { + ServerName string + + // Failures is how many sends in a row have failed. Zero is a healthy + // destination. + Failures int + + // RetryAtMS is when the next attempt may be made; zero means now. + RetryAtMS int64 + + LastSuccessMS int64 + LastFailureMS int64 + LastError string +} + +// DestinationSummary is a destination with the size of its queue, for the +// admin API. +type DestinationSummary struct { + Destination + + QueuedPDUs int + QueuedEDUs int +} + +// FederationStore persists what federation needs to survive a restart: +// other servers' keys, the outbound queue, the per-destination backoff and +// the log of inbound transactions. +type FederationStore interface { + // StoreServerKeys inserts or replaces keys. A key already stored is + // overwritten with the newer report of it. + StoreServerKeys(ctx context.Context, keys []ServerKey) error + + // ServerKeys returns the stored keys of a server. An empty keyIDs + // returns all of them. + ServerKeys(ctx context.Context, serverName string, keyIDs []string) ([]ServerKey, error) + + // Enqueue adds items to the outbound queue, assigning their IDs. + Enqueue(ctx context.Context, items []QueuedItem) ([]QueuedItem, error) + + // Queued returns up to limit of a destination's items of one kind, + // oldest first. + Queued(ctx context.Context, destination, kind string, limit int) ([]QueuedItem, error) + + // Dequeue removes delivered items. + Dequeue(ctx context.Context, ids []int64) error + + // QueuedDestinations lists the destinations that have anything queued, + // which is where a restarted queue picks up. + QueuedDestinations(ctx context.Context) ([]string, error) + + // Destination reads one destination's delivery health. A server that + // has never been sent anything comes back as a zero Destination with + // its name filled in and no error. + Destination(ctx context.Context, serverName string) (Destination, error) + + // SetDestination records a destination's delivery health. + SetDestination(ctx context.Context, d Destination) error + + // Destinations lists every destination known to the queue, with its + // queue depth, ordered by server name. + Destinations(ctx context.Context) ([]DestinationSummary, error) + + // InboundTransaction returns the response recorded for a transaction + // already received from origin, and whether there was one. + InboundTransaction(ctx context.Context, origin, txnID string) ([]byte, bool, error) + + // RecordInboundTransaction stores the response to a transaction so + // that a retransmission is answered the same way. + RecordInboundTransaction(ctx context.Context, origin, txnID string, response []byte, atMS int64) error +} diff --git a/server/internal/storage/federation_test.go b/server/internal/storage/federation_test.go new file mode 100644 index 0000000..f79f3a1 --- /dev/null +++ b/server/internal/storage/federation_test.go @@ -0,0 +1,431 @@ +package storage_test + +import ( + "database/sql" + "os" + "slices" + "testing" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// migratedFederationDB opens a driver with the room and federation tables +// empty. +func migratedFederationDB(t *testing.T, driver string) storage.Database { + t.Helper() + db := migratedRoomDB(t, driver) + if driver == storage.DriverPostgres { + pool, err := sql.Open("pgx", os.Getenv("WARP_TEST_POSTGRES_DSN")) + if err != nil { + t.Fatalf("open postgres to reset the federation tables: %v", err) + } + defer func() { _ = pool.Close() }() + if _, err := pool.ExecContext(t.Context(), `TRUNCATE + federation_server_keys, federation_queue, + federation_destinations, federation_inbound_txns`); err != nil { + t.Fatalf("reset the federation tables: %v", err) + } + if _, err := pool.ExecContext(t.Context(), `UPDATE federation_sequences SET value = 0`); err != nil { + t.Fatalf("rewind the federation counters: %v", err) + } + } + return db +} + +func TestOutliersAreReadableButNotInTheTimeline(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + rooms := migratedFederationDB(t, driver).Rooms() + if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { + t.Fatalf("CreateRoom: %v", err) + } + + outliers := []storage.StoredEvent{ + newTestEvent(testRoomID, "$create", "m.room.create", stateKey("")), + newTestEvent(testRoomID, "$member", "m.room.member", stateKey("@bob:remote.example")), + } + if err := rooms.WriteOutliers(ctx, testRoomID, outliers); err != nil { + t.Fatalf("WriteOutliers: %v", err) + } + // Writing them again is not an error: an event ID names its + // contents, so the second copy is the first one. + if err := rooms.WriteOutliers(ctx, testRoomID, outliers); err != nil { + t.Fatalf("WriteOutliers twice: %v", err) + } + + got, err := rooms.EventsByID(ctx, testRoomID, []string{"$create", "$member"}) + if err != nil { + t.Fatalf("EventsByID: %v", err) + } + if len(got) != 2 { + t.Fatalf("EventsByID found %d outliers, want 2", len(got)) + } + for _, e := range got { + if !e.Outlier { + t.Errorf("%s came back as a timeline event", e.EventID) + } + } + + // A join that cites the outliers as its state. + join := newTestEvent(testRoomID, "$join", "m.room.member", stateKey("@alice:warp.example")) + if _, err := rooms.Write(ctx, storage.RoomWrite{ + Event: join, + StateBefore: []storage.StateEntry{ + {Type: "m.room.create", EventID: "$create"}, + {Type: "m.room.member", StateKey: "@bob:remote.example", EventID: "$member", Membership: "join"}, + }, + StateDelta: []storage.StateEntry{ + {Type: "m.room.create", EventID: "$create"}, + {Type: "m.room.member", StateKey: "@bob:remote.example", EventID: "$member", Membership: "join"}, + {Type: "m.room.member", StateKey: "@alice:warp.example", EventID: "$join", Membership: "join"}, + }, + StateDeltaIsWholeState: true, + Extremities: []string{"$join"}, + }); err != nil { + t.Fatalf("Write join: %v", err) + } + + timeline, err := rooms.Timeline(ctx, storage.TimelineRequest{ + RoomID: testRoomID, Limit: 10, Direction: storage.DirectionBackward, + }) + if err != nil { + t.Fatalf("Timeline: %v", err) + } + if ids := eventIDsOf(timeline); !slices.Equal(ids, []string{"$join"}) { + t.Errorf("timeline = %v, want only the join", ids) + } + + // The current state reads through to the outliers: that is + // the whole point of keeping them in the events table. + state, err := rooms.CurrentStateEvents(ctx, testRoomID) + if err != nil { + t.Fatalf("CurrentStateEvents: %v", err) + } + if len(state) != 3 { + t.Errorf("current state has %d events, want 3", len(state)) + } + + top, err := rooms.MaxStreamOrdering(ctx) + if err != nil { + t.Fatalf("MaxStreamOrdering: %v", err) + } + at, err := rooms.StateAtStream(ctx, testRoomID, top) + if err != nil { + t.Fatalf("StateAtStream: %v", err) + } + if len(at) != 3 { + t.Errorf("state at the join has %d entries, want 3", len(at)) + } + }) + } +} + +func TestOutlierIsPromotedWhenItArrivesAgainAsAnEvent(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + rooms := migratedFederationDB(t, driver).Rooms() + if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { + t.Fatalf("CreateRoom: %v", err) + } + writeMessages(t, rooms, testRoomID, "$first") + if err := rooms.WriteOutliers(ctx, testRoomID, []storage.StoredEvent{ + newTestEvent(testRoomID, "$late", "m.room.message", nil), + }); err != nil { + t.Fatalf("WriteOutliers: %v", err) + } + writeMessages(t, rooms, testRoomID, "$late") + + stored, err := rooms.Event(ctx, testRoomID, "$late") + if err != nil { + t.Fatalf("Event: %v", err) + } + if stored.Outlier { + t.Error("the event is still an outlier after being written as one") + } + timeline, err := rooms.Timeline(ctx, storage.TimelineRequest{ + RoomID: testRoomID, Limit: 10, Direction: storage.DirectionForward, + }) + if err != nil { + t.Fatalf("Timeline: %v", err) + } + if ids := eventIDsOf(timeline); !slices.Equal(ids, []string{"$first", "$late"}) { + t.Errorf("timeline = %v, want [$first $late]", ids) + } + }) + } +} + +func TestInviteStateLastsUntilTheNextMembershipChange(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + rooms := migratedFederationDB(t, driver).Rooms() + if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { + t.Fatalf("CreateRoom: %v", err) + } + const invitee = "@alice:warp.example" + stripped := []byte(`[{"type":"m.room.name","state_key":"","sender":"@bob:remote.example","content":{"name":"Hi"}}]`) + + invite := newTestEvent(testRoomID, "$invite", "m.room.member", stateKey(invitee)) + if _, err := rooms.Write(ctx, storage.RoomWrite{ + Event: invite, + Outlier: true, + StateDelta: []storage.StateEntry{{Type: "m.room.member", StateKey: invitee, EventID: "$invite", Membership: "invite"}}, + InviteState: stripped, + }); err != nil { + t.Fatalf("Write invite: %v", err) + } + + got, err := rooms.InviteState(ctx, testRoomID, invitee) + if err != nil { + t.Fatalf("InviteState: %v", err) + } + if string(got) != string(stripped) { + t.Errorf("InviteState = %s, want %s", got, stripped) + } + memberships, err := rooms.MembershipsForUser(ctx, invitee, []string{"invite"}) + if err != nil { + t.Fatalf("MembershipsForUser: %v", err) + } + if len(memberships) != 1 || memberships[0].StreamOrdering == 0 { + t.Errorf("an outlier invite must still have a stream position for /sync, got %+v", memberships) + } + + leave := newTestEvent(testRoomID, "$leave", "m.room.member", stateKey(invitee)) + if _, err := rooms.Write(ctx, storage.RoomWrite{ + Event: leave, + Outlier: true, + StateDelta: []storage.StateEntry{{Type: "m.room.member", StateKey: invitee, EventID: "$leave", Membership: "leave"}}, + }); err != nil { + t.Fatalf("Write leave: %v", err) + } + if got, err := rooms.InviteState(ctx, testRoomID, invitee); err != nil || got != nil { + t.Errorf("InviteState after a rejection = %s, %v; want nothing", got, err) + } + }) + } +} + +func TestServerKeysAreUpsertedAndFiltered(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + fed := migratedFederationDB(t, driver).Federation() + + if err := fed.StoreServerKeys(ctx, []storage.ServerKey{ + {ServerName: "remote.example", KeyID: "ed25519:a", PublicKey: "AAA", ValidUntilTS: 10, FetchedAtMS: 1}, + {ServerName: "remote.example", KeyID: "ed25519:b", PublicKey: "BBB", ValidUntilTS: 20, ExpiredTS: 15, FetchedAtMS: 1}, + {ServerName: "other.example", KeyID: "ed25519:a", PublicKey: "CCC", ValidUntilTS: 30, FetchedAtMS: 1}, + }); err != nil { + t.Fatalf("StoreServerKeys: %v", err) + } + if err := fed.StoreServerKeys(ctx, []storage.ServerKey{ + {ServerName: "remote.example", KeyID: "ed25519:a", PublicKey: "AAA", ValidUntilTS: 99, FetchedAtMS: 2}, + }); err != nil { + t.Fatalf("StoreServerKeys again: %v", err) + } + + all, err := fed.ServerKeys(ctx, "remote.example", nil) + if err != nil { + t.Fatalf("ServerKeys: %v", err) + } + if len(all) != 2 { + t.Fatalf("ServerKeys found %d keys, want 2", len(all)) + } + if all[0].KeyID != "ed25519:a" || all[0].ValidUntilTS != 99 { + t.Errorf("the refetched key was not replaced: %+v", all[0]) + } + if all[1].ExpiredTS != 15 { + t.Errorf("an old key lost its expiry: %+v", all[1]) + } + + one, err := fed.ServerKeys(ctx, "remote.example", []string{"ed25519:b"}) + if err != nil { + t.Fatalf("ServerKeys(b): %v", err) + } + if len(one) != 1 || one[0].PublicKey != "BBB" { + t.Errorf("ServerKeys(b) = %+v", one) + } + }) + } +} + +func TestQueueIsPerDestinationAndInOrder(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + fed := migratedFederationDB(t, driver).Federation() + + queued, err := fed.Enqueue(ctx, []storage.QueuedItem{ + {Destination: "a.example", Kind: storage.QueueKindPDU, EventID: "$1", Payload: []byte(`{"n":1}`)}, + {Destination: "b.example", Kind: storage.QueueKindPDU, EventID: "$1", Payload: []byte(`{"n":1}`)}, + {Destination: "a.example", Kind: storage.QueueKindEDU, Payload: []byte(`{"edu":true}`)}, + {Destination: "a.example", Kind: storage.QueueKindPDU, EventID: "$2", Payload: []byte(`{"n":2}`)}, + }) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + for i := 1; i < len(queued); i++ { + if queued[i].ID <= queued[i-1].ID { + t.Fatalf("queue IDs are not increasing: %d then %d", queued[i-1].ID, queued[i].ID) + } + } + + pdus, err := fed.Queued(ctx, "a.example", storage.QueueKindPDU, 10) + if err != nil { + t.Fatalf("Queued: %v", err) + } + if len(pdus) != 2 || pdus[0].EventID != "$1" || pdus[1].EventID != "$2" { + t.Fatalf("Queued(a, pdu) = %+v", pdus) + } + if string(pdus[1].Payload) != `{"n":2}` { + t.Errorf("payload = %s", pdus[1].Payload) + } + + destinations, err := fed.QueuedDestinations(ctx) + if err != nil { + t.Fatalf("QueuedDestinations: %v", err) + } + if !slices.Equal(destinations, []string{"a.example", "b.example"}) { + t.Errorf("QueuedDestinations = %v", destinations) + } + + if err := fed.Dequeue(ctx, []int64{pdus[0].ID}); err != nil { + t.Fatalf("Dequeue: %v", err) + } + left, err := fed.Queued(ctx, "a.example", storage.QueueKindPDU, 10) + if err != nil { + t.Fatalf("Queued after Dequeue: %v", err) + } + if len(left) != 1 || left[0].EventID != "$2" { + t.Errorf("after dequeue = %+v", left) + } + }) + } +} + +func TestDestinationHealth(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + fed := migratedFederationDB(t, driver).Federation() + + d, err := fed.Destination(ctx, "never.example") + if err != nil { + t.Fatalf("Destination: %v", err) + } + if d.ServerName != "never.example" || d.Failures != 0 { + t.Errorf("an unknown destination = %+v, want a healthy zero", d) + } + + if err := fed.SetDestination(ctx, storage.Destination{ + ServerName: "down.example", Failures: 3, RetryAtMS: 500, LastError: "connection refused", + }); err != nil { + t.Fatalf("SetDestination: %v", err) + } + if err := fed.SetDestination(ctx, storage.Destination{ + ServerName: "down.example", Failures: 4, RetryAtMS: 900, LastError: "timeout", + }); err != nil { + t.Fatalf("SetDestination again: %v", err) + } + if _, err := fed.Enqueue(ctx, []storage.QueuedItem{ + {Destination: "busy.example", Kind: storage.QueueKindPDU, Payload: []byte(`{}`)}, + {Destination: "busy.example", Kind: storage.QueueKindEDU, Payload: []byte(`{}`)}, + {Destination: "down.example", Kind: storage.QueueKindPDU, Payload: []byte(`{}`)}, + }); err != nil { + t.Fatalf("Enqueue: %v", err) + } + + all, err := fed.Destinations(ctx) + if err != nil { + t.Fatalf("Destinations: %v", err) + } + if len(all) != 2 { + t.Fatalf("Destinations = %+v, want busy and down", all) + } + busy, down := all[0], all[1] + if busy.ServerName != "busy.example" || busy.QueuedPDUs != 1 || busy.QueuedEDUs != 1 || busy.Failures != 0 { + t.Errorf("busy = %+v", busy) + } + if down.ServerName != "down.example" || down.Failures != 4 || down.LastError != "timeout" || down.QueuedPDUs != 1 { + t.Errorf("down = %+v", down) + } + }) + } +} + +func TestInboundTransactionsAreRememberedOnce(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + fed := migratedFederationDB(t, driver).Federation() + + if _, ok, err := fed.InboundTransaction(ctx, "remote.example", "t1"); err != nil || ok { + t.Fatalf("InboundTransaction before = %v, %v", ok, err) + } + if err := fed.RecordInboundTransaction(ctx, "remote.example", "t1", []byte(`{"pdus":{}}`), 1); err != nil { + t.Fatalf("RecordInboundTransaction: %v", err) + } + if err := fed.RecordInboundTransaction(ctx, "remote.example", "t1", []byte(`{"other":1}`), 2); err != nil { + t.Fatalf("RecordInboundTransaction again: %v", err) + } + got, ok, err := fed.InboundTransaction(ctx, "remote.example", "t1") + if err != nil || !ok || string(got) != `{"pdus":{}}` { + t.Errorf("InboundTransaction = %s, %v, %v; want the first response", got, ok, err) + } + }) + } +} + +func eventIDsOf(events []storage.StoredEvent) []string { + out := make([]string, 0, len(events)) + for _, e := range events { + out = append(out, e.EventID) + } + return out +} + +func TestHasJoinedMemberOn(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + rooms := migratedFederationDB(t, driver).Rooms() + if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { + t.Fatal(err) + } + for _, m := range []struct{ user, membership string }{ + {"@a:one.example", "join"}, + {"@b:two.example", "leave"}, + {"@c:x_y.example", "join"}, + } { + e := newTestEvent(testRoomID, "$"+m.user, "m.room.member", stateKey(m.user)) + if _, err := rooms.Write(ctx, storage.RoomWrite{ + Event: e, + StateDelta: []storage.StateEntry{{Type: "m.room.member", StateKey: m.user, EventID: e.EventID, Membership: m.membership}}, + Extremities: []string{e.EventID}, + }); err != nil { + t.Fatal(err) + } + } + cases := map[string]bool{ + "one.example": true, + "two.example": false, // left + "x_y.example": true, + "xay.example": false, // '_' is not a LIKE wildcard here + "e.example": false, // a suffix of a name is not the name + "none.example": false, + } + for server, want := range cases { + got, err := rooms.HasJoinedMemberOn(ctx, testRoomID, server) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("HasJoinedMemberOn(%s) = %v, want %v", server, got, want) + } + } + }) + } +} diff --git a/server/internal/storage/pocketbase/pocketbase.go b/server/internal/storage/pocketbase/pocketbase.go index e2251e9..56c0a5d 100644 --- a/server/internal/storage/pocketbase/pocketbase.go +++ b/server/internal/storage/pocketbase/pocketbase.go @@ -60,6 +60,9 @@ func (*Database) Rooms() storage.RoomStore { return notImplementedRooms{} } // Sync implements storage.Database. func (*Database) Sync() storage.SyncStore { return notImplementedSync{} } +// Federation implements storage.Database. +func (*Database) Federation() storage.FederationStore { return notImplementedFederation{} } + // Close implements storage.Database. func (*Database) Close() error { return nil } @@ -298,6 +301,75 @@ func (notImplementedRooms) RoomStats(context.Context) (storage.RoomStats, error) return storage.RoomStats{}, storage.ErrNotImplemented } +func (notImplementedRooms) WriteOutliers(context.Context, string, []storage.StoredEvent) error { + return storage.ErrNotImplemented +} + +func (notImplementedRooms) WriteBackfill(context.Context, string, []storage.BackfillEvent) error { + return storage.ErrNotImplemented +} + +func (notImplementedRooms) InviteState(context.Context, string, string) ([]byte, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedRooms) HasJoinedMemberOn(context.Context, string, string) (bool, error) { + return false, storage.ErrNotImplemented +} + +func (notImplementedRooms) EventByID(context.Context, string) (storage.StoredEvent, error) { + return storage.StoredEvent{}, storage.ErrNotImplemented +} + +// notImplementedFederation satisfies storage.FederationStore the same way. +// A queue that accepted events and never stored them would lose them the +// moment federation was switched on. +type notImplementedFederation struct{} + +func (notImplementedFederation) StoreServerKeys(context.Context, []storage.ServerKey) error { + return storage.ErrNotImplemented +} + +func (notImplementedFederation) ServerKeys(context.Context, string, []string) ([]storage.ServerKey, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedFederation) Enqueue(context.Context, []storage.QueuedItem) ([]storage.QueuedItem, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedFederation) Queued(context.Context, string, string, int) ([]storage.QueuedItem, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedFederation) Dequeue(context.Context, []int64) error { + return storage.ErrNotImplemented +} + +func (notImplementedFederation) QueuedDestinations(context.Context) ([]string, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedFederation) Destination(context.Context, string) (storage.Destination, error) { + return storage.Destination{}, storage.ErrNotImplemented +} + +func (notImplementedFederation) SetDestination(context.Context, storage.Destination) error { + return storage.ErrNotImplemented +} + +func (notImplementedFederation) Destinations(context.Context) ([]storage.DestinationSummary, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedFederation) InboundTransaction(context.Context, string, string) ([]byte, bool, error) { + return nil, false, storage.ErrNotImplemented +} + +func (notImplementedFederation) RecordInboundTransaction(context.Context, string, string, []byte, int64) error { + return storage.ErrNotImplemented +} + // notImplementedSync satisfies storage.SyncStore the same way. A sync that // returned an empty success would be worse than an error: a client would // take it for "nothing has happened" and stop asking. @@ -361,8 +433,9 @@ func (notImplementedSync) Filter(context.Context, string, string) ([]byte, error // Compile-time proof that the stub keeps up with the interfaces. var ( - _ storage.Database = (*Database)(nil) - _ storage.UserStore = notImplementedUsers{} - _ storage.RoomStore = notImplementedRooms{} - _ storage.SyncStore = notImplementedSync{} + _ storage.Database = (*Database)(nil) + _ storage.UserStore = notImplementedUsers{} + _ storage.RoomStore = notImplementedRooms{} + _ storage.SyncStore = notImplementedSync{} + _ storage.FederationStore = notImplementedFederation{} ) diff --git a/server/internal/storage/postgres/migrations/0005_federation.sql b/server/internal/storage/postgres/migrations/0005_federation.sql new file mode 100644 index 0000000..73b7fb1 --- /dev/null +++ b/server/internal/storage/postgres/migrations/0005_federation.sql @@ -0,0 +1,113 @@ +-- Federation: other servers' keys, the outbound queue and its per-server +-- backoff, the inbound transaction log, and the two additions the room +-- tables need once a room's events can come from somewhere else. +-- +-- Timestamps are milliseconds since the Unix epoch, as everywhere else in +-- Warp. + +-- An outlier is an event this server holds without holding the room's +-- state at it: the state and auth chain a join is handed by the resident +-- server, an auth event fetched to judge something else, an invite to a +-- room nobody here is in, an event that soft-failed. It is a real event — +-- it is signed, it was authorised against its own auth events, and other +-- events can cite it — but it has no place in the timeline: no state +-- snapshot (state_before_id stays 0), never a forward extremity, never +-- returned by /sync or /messages. +-- +-- It goes on the events table rather than into a table of its own because +-- every read of a room's current state joins the events table, and the +-- current state of a room joined over federation is made almost entirely +-- of outliers. An outlier that later arrives as an ordinary event — fetched +-- by /get_missing_events, say — is promoted in place. +ALTER TABLE roomserver_events ADD COLUMN outlier BOOLEAN NOT NULL DEFAULT FALSE; + +-- The stripped state an invite over federation arrives with. +-- +-- An invite to a room this server is not in is the invite event alone: the +-- room's state is on the inviting server. What a client shows in its +-- invite list — the room's name, its avatar, who invited you — comes from +-- the invite_room_state the inviting server sent along, which is stored +-- here until the invite is answered and then deleted with it. +CREATE TABLE IF NOT EXISTS roomserver_invite_state ( + room_id TEXT NOT NULL REFERENCES roomserver_rooms (room_id) ON DELETE CASCADE, + user_id TEXT NOT NULL, + -- A JSON array of stripped state events, exactly as received. + state_json TEXT NOT NULL, + PRIMARY KEY (room_id, user_id) +); + +-- Other servers' signing keys, as their key servers or a notary reported +-- them. A key is never deleted: an event signed years ago is still +-- verified with the key that signed it, and a server that has rotated its +-- key lists the old one only as an old_verify_key with the time it expired. +CREATE TABLE IF NOT EXISTS federation_server_keys ( + server_name TEXT NOT NULL, + key_id TEXT NOT NULL, + -- Unpadded base64, as it is on the wire. + public_key TEXT NOT NULL, + -- How long the server vouched for the key when it was fetched. A key + -- is re-fetched once this passes, and an event from room version 5 on + -- may not be verified with a key whose validity ended before the event + -- was sent. + valid_until_ts BIGINT NOT NULL, + -- When the key stopped being used, for an old_verify_key; 0 for a key + -- the server still signs with. + expired_ts BIGINT NOT NULL DEFAULT 0, + fetched_at_ms BIGINT NOT NULL, + PRIMARY KEY (server_name, key_id) +); + +-- One counter row per stream; see roomserver_sequences for why these are +-- rows and not database sequences. +CREATE TABLE IF NOT EXISTS federation_sequences ( + name TEXT NOT NULL PRIMARY KEY, + value BIGINT NOT NULL +); + +INSERT INTO federation_sequences (name, value) VALUES ('queue', 0) + ON CONFLICT (name) DO NOTHING; + +-- The outbound queue: every PDU and EDU waiting for a destination. +-- +-- A row is deleted once the destination has acknowledged the transaction +-- that carried it, so what is here after a restart is exactly what still +-- has to be sent. The payload is the JSON that goes on the wire, copied at +-- enqueue time: re-reading a PDU from the room tables at send time would +-- send its redacted form if it was redacted in the meantime, and that is +-- not the event the destination has to verify. +CREATE TABLE IF NOT EXISTS federation_queue ( + queue_id BIGINT NOT NULL PRIMARY KEY, + destination TEXT NOT NULL, + -- 'pdu' or 'edu'. + kind TEXT NOT NULL, + room_id TEXT NOT NULL DEFAULT '', + event_id TEXT NOT NULL DEFAULT '', + payload TEXT NOT NULL, + queued_at_ms BIGINT NOT NULL +); + +CREATE INDEX IF NOT EXISTS federation_queue_destination_idx + ON federation_queue (destination, kind, queue_id); + +-- Per-destination delivery health: how often sending has failed in a row +-- and when it may next be tried. A destination with no row has never +-- failed. +CREATE TABLE IF NOT EXISTS federation_destinations ( + server_name TEXT NOT NULL PRIMARY KEY, + failures INTEGER NOT NULL DEFAULT 0, + retry_at_ms BIGINT NOT NULL DEFAULT 0, + last_success_ms BIGINT NOT NULL DEFAULT 0, + last_failure_ms BIGINT NOT NULL DEFAULT 0, + last_error TEXT NOT NULL DEFAULT '' +); + +-- Transactions received from other servers, so that a retransmission is +-- answered with the response the first delivery got instead of being +-- processed a second time. +CREATE TABLE IF NOT EXISTS federation_inbound_txns ( + origin TEXT NOT NULL, + txn_id TEXT NOT NULL, + response_json TEXT NOT NULL, + received_at_ms BIGINT NOT NULL, + PRIMARY KEY (origin, txn_id) +); diff --git a/server/internal/storage/postgres/migrations/0006_backfill.sql b/server/internal/storage/postgres/migrations/0006_backfill.sql new file mode 100644 index 0000000..b33e307 --- /dev/null +++ b/server/internal/storage/postgres/migrations/0006_backfill.sql @@ -0,0 +1,15 @@ +-- Outbound backfill: history fetched from other servers for a room this +-- server joined part-way through. +-- +-- roomserver_events.stream_ordering is the order clients are shown a room +-- in, and the stream_ordering counter only ever goes up: every event it +-- numbers arrived after the ones before it. Backfilled history arrives +-- after everything too, but belongs before all of it. So it is numbered by +-- a second counter that runs the other way — from -1 downwards — and every +-- backfilled event sits below every event this server held when it came. +-- +-- Zero is neither counter's: the stream_ordering counter's first value is +-- 1 and this one's is -1, and a window bound of zero keeps meaning +-- "unbounded at that end". +INSERT INTO roomserver_sequences (name, value) VALUES ('backfill_ordering', 0) + ON CONFLICT (name) DO NOTHING; diff --git a/server/internal/storage/rooms.go b/server/internal/storage/rooms.go index bb2872b..82d8723 100644 --- a/server/internal/storage/rooms.go +++ b/server/internal/storage/rooms.go @@ -120,9 +120,12 @@ type StoredEvent struct { // JSON is the event as canonical JSON, exactly as it was signed. JSON []byte - // StreamOrdering is this server's own monotonic position for the event: - // the order it was persisted in, which is the order /sync and /messages - // hand events to clients. + // StreamOrdering is this server's own position for the event: the + // order /sync and /messages hand events to clients. For an event that + // arrived live it is the order it was persisted in, counting up from 1; + // for history backfilled from another server it is negative, counting + // down from -1, so that it sorts below everything that was already + // here. It is never zero. StreamOrdering int64 // RedactedBy is the event ID of the redaction that applies to this @@ -130,6 +133,13 @@ type StoredEvent struct { // applied, so a redacted event is already stripped on the way out; this // field is what lets unsigned.redacted_because be filled in. RedactedBy string + + // Outlier reports an event held without the room's state at it: part + // of a join's state or auth chain, an auth event fetched to judge + // another, an invite to a room nobody here is in, or an event that + // soft-failed. An outlier can be cited and read by ID, and is never in + // a timeline. + Outlier bool } // IsState reports whether the event is a state event. @@ -193,6 +203,32 @@ type RoomWrite struct { // replaces the stored JSON. Redacts string RedactedJSON []byte + + // Outlier writes the event as an outlier: no state snapshot is + // recorded for it and StateBefore is ignored. StateDelta and + // Extremities still apply when they are set, which is how an invite to + // a room nobody here is in becomes the one piece of that room's state + // this server knows. + Outlier bool + + // InviteState is the stripped state an invite over federation arrived + // with, kept for the invitee until the invite is answered. Any + // m.room.member write clears what was stored for its state key first, + // so the invite's own write is the only one that leaves a row behind. + InviteState []byte +} + +// BackfillEvent is one event of a room's history, fetched from another +// server after this server already held later events, with the room state +// it was authorised against. +type BackfillEvent struct { + Event StoredEvent + + // StateBefore is the state of the room as of the event's prev_events, + // recorded as its state snapshot exactly as RoomWrite.StateBefore is. + // It is what lets a client paging back through backfilled history be + // told the state at any point of it. + StateBefore []StateEntry } // RoomAlias is a local alias and the room it points at. @@ -342,7 +378,9 @@ type RoomStore interface { RoomsWithEvents(ctx context.Context, roomIDs []string, from, to int64) ([]string, error) // StateAtStream returns the room's state as of a stream position: the - // state after the last event at or before pos. + // state after the last event at or before pos. The position may be a + // negative one inside backfilled history; zero has no events at or + // before it that mean anything and returns nothing. // // It answers the two questions current state cannot. A user who has // left sees the room as it was when they left, and a sync with a gap in @@ -393,4 +431,39 @@ type RoomStore interface { // RoomStats returns the aggregate counts for the admin dashboard. RoomStats(ctx context.Context) (RoomStats, error) + + // WriteOutliers stores events as outliers in one transaction, skipping + // any already held. It is how a join over federation stores the room + // state and auth chain it is handed, which can run to tens of thousands + // of events: one Write per event would be one transaction per event. + WriteOutliers(ctx context.Context, roomID string, events []StoredEvent) error + + // WriteBackfill stores a room's history, fetched from another server, + // below everything the room already holds, in one transaction. + // + // The events are given oldest first and keep that order: each is + // numbered by a counter that runs downwards from -1, so the whole batch + // sorts below every event this server held when it arrived, and a + // later batch sorts below this one. An event already in the timeline is + // skipped; an outlier is promoted into the timeline in place, keeping + // its stored JSON. Nothing else about the room changes — not its + // current state, not its forward extremities, not its member counts — + // because history is by definition already reflected in them. + WriteBackfill(ctx context.Context, roomID string, events []BackfillEvent) error + + // InviteState returns the stripped state stored with a user's invite, + // or nil when there is none. + InviteState(ctx context.Context, roomID, userID string) ([]byte, error) + + // HasJoinedMemberOn reports whether any user of a server is joined to + // a room. It is asked about every event that arrives over federation — + // is this server still in the room, is the sender's? — and answering + // it by listing the members would read every row of a room of tens of + // thousands. + HasJoinedMemberOn(ctx context.Context, roomID, serverName string) (bool, error) + + // EventByID reads an event without knowing its room, returning + // ErrNoSuchEvent if it is absent. The federation API's /event names + // only the event; everything else asks within a room. + EventByID(ctx context.Context, eventID string) (StoredEvent, error) } diff --git a/server/internal/storage/rooms_test.go b/server/internal/storage/rooms_test.go index 9c7ea7f..9240989 100644 --- a/server/internal/storage/rooms_test.go +++ b/server/internal/storage/rooms_test.go @@ -41,7 +41,7 @@ func resetRoomTables(t *testing.T, driver string) { if _, err := db.ExecContext(t.Context(), `TRUNCATE roomserver_state_entries, roomserver_state_snapshots, roomserver_current_state, roomserver_extremities, - roomserver_transactions, roomserver_aliases, + roomserver_transactions, roomserver_aliases, roomserver_invite_state, roomserver_events, roomserver_rooms`); err != nil { t.Fatalf("reset the room tables: %v", err) } diff --git a/server/internal/storage/sqlite/migrations/0005_federation.sql b/server/internal/storage/sqlite/migrations/0005_federation.sql new file mode 100644 index 0000000..7d7732d --- /dev/null +++ b/server/internal/storage/sqlite/migrations/0005_federation.sql @@ -0,0 +1,113 @@ +-- Federation: other servers' keys, the outbound queue and its per-server +-- backoff, the inbound transaction log, and the two additions the room +-- tables need once a room's events can come from somewhere else. +-- +-- Timestamps are milliseconds since the Unix epoch, as everywhere else in +-- Warp. + +-- An outlier is an event this server holds without holding the room's +-- state at it: the state and auth chain a join is handed by the resident +-- server, an auth event fetched to judge something else, an invite to a +-- room nobody here is in, an event that soft-failed. It is a real event — +-- it is signed, it was authorised against its own auth events, and other +-- events can cite it — but it has no place in the timeline: no state +-- snapshot (state_before_id stays 0), never a forward extremity, never +-- returned by /sync or /messages. +-- +-- It goes on the events table rather than into a table of its own because +-- every read of a room's current state joins the events table, and the +-- current state of a room joined over federation is made almost entirely +-- of outliers. An outlier that later arrives as an ordinary event — fetched +-- by /get_missing_events, say — is promoted in place. +ALTER TABLE roomserver_events ADD COLUMN outlier BOOLEAN NOT NULL DEFAULT 0; + +-- The stripped state an invite over federation arrives with. +-- +-- An invite to a room this server is not in is the invite event alone: the +-- room's state is on the inviting server. What a client shows in its +-- invite list — the room's name, its avatar, who invited you — comes from +-- the invite_room_state the inviting server sent along, which is stored +-- here until the invite is answered and then deleted with it. +CREATE TABLE IF NOT EXISTS roomserver_invite_state ( + room_id TEXT NOT NULL REFERENCES roomserver_rooms (room_id) ON DELETE CASCADE, + user_id TEXT NOT NULL, + -- A JSON array of stripped state events, exactly as received. + state_json TEXT NOT NULL, + PRIMARY KEY (room_id, user_id) +); + +-- Other servers' signing keys, as their key servers or a notary reported +-- them. A key is never deleted: an event signed years ago is still +-- verified with the key that signed it, and a server that has rotated its +-- key lists the old one only as an old_verify_key with the time it expired. +CREATE TABLE IF NOT EXISTS federation_server_keys ( + server_name TEXT NOT NULL, + key_id TEXT NOT NULL, + -- Unpadded base64, as it is on the wire. + public_key TEXT NOT NULL, + -- How long the server vouched for the key when it was fetched. A key + -- is re-fetched once this passes, and an event from room version 5 on + -- may not be verified with a key whose validity ended before the event + -- was sent. + valid_until_ts BIGINT NOT NULL, + -- When the key stopped being used, for an old_verify_key; 0 for a key + -- the server still signs with. + expired_ts BIGINT NOT NULL DEFAULT 0, + fetched_at_ms BIGINT NOT NULL, + PRIMARY KEY (server_name, key_id) +); + +-- One counter row per stream; see roomserver_sequences for why these are +-- rows and not database sequences. +CREATE TABLE IF NOT EXISTS federation_sequences ( + name TEXT NOT NULL PRIMARY KEY, + value BIGINT NOT NULL +); + +INSERT INTO federation_sequences (name, value) VALUES ('queue', 0) + ON CONFLICT (name) DO NOTHING; + +-- The outbound queue: every PDU and EDU waiting for a destination. +-- +-- A row is deleted once the destination has acknowledged the transaction +-- that carried it, so what is here after a restart is exactly what still +-- has to be sent. The payload is the JSON that goes on the wire, copied at +-- enqueue time: re-reading a PDU from the room tables at send time would +-- send its redacted form if it was redacted in the meantime, and that is +-- not the event the destination has to verify. +CREATE TABLE IF NOT EXISTS federation_queue ( + queue_id BIGINT NOT NULL PRIMARY KEY, + destination TEXT NOT NULL, + -- 'pdu' or 'edu'. + kind TEXT NOT NULL, + room_id TEXT NOT NULL DEFAULT '', + event_id TEXT NOT NULL DEFAULT '', + payload TEXT NOT NULL, + queued_at_ms BIGINT NOT NULL +); + +CREATE INDEX IF NOT EXISTS federation_queue_destination_idx + ON federation_queue (destination, kind, queue_id); + +-- Per-destination delivery health: how often sending has failed in a row +-- and when it may next be tried. A destination with no row has never +-- failed. +CREATE TABLE IF NOT EXISTS federation_destinations ( + server_name TEXT NOT NULL PRIMARY KEY, + failures INTEGER NOT NULL DEFAULT 0, + retry_at_ms BIGINT NOT NULL DEFAULT 0, + last_success_ms BIGINT NOT NULL DEFAULT 0, + last_failure_ms BIGINT NOT NULL DEFAULT 0, + last_error TEXT NOT NULL DEFAULT '' +); + +-- Transactions received from other servers, so that a retransmission is +-- answered with the response the first delivery got instead of being +-- processed a second time. +CREATE TABLE IF NOT EXISTS federation_inbound_txns ( + origin TEXT NOT NULL, + txn_id TEXT NOT NULL, + response_json TEXT NOT NULL, + received_at_ms BIGINT NOT NULL, + PRIMARY KEY (origin, txn_id) +); diff --git a/server/internal/storage/sqlite/migrations/0006_backfill.sql b/server/internal/storage/sqlite/migrations/0006_backfill.sql new file mode 100644 index 0000000..b33e307 --- /dev/null +++ b/server/internal/storage/sqlite/migrations/0006_backfill.sql @@ -0,0 +1,15 @@ +-- Outbound backfill: history fetched from other servers for a room this +-- server joined part-way through. +-- +-- roomserver_events.stream_ordering is the order clients are shown a room +-- in, and the stream_ordering counter only ever goes up: every event it +-- numbers arrived after the ones before it. Backfilled history arrives +-- after everything too, but belongs before all of it. So it is numbered by +-- a second counter that runs the other way — from -1 downwards — and every +-- backfilled event sits below every event this server held when it came. +-- +-- Zero is neither counter's: the stream_ordering counter's first value is +-- 1 and this one's is -1, and a window bound of zero keeps meaning +-- "unbounded at that end". +INSERT INTO roomserver_sequences (name, value) VALUES ('backfill_ordering', 0) + ON CONFLICT (name) DO NOTHING; diff --git a/server/internal/storage/sqlstore/federation.go b/server/internal/storage/sqlstore/federation.go new file mode 100644 index 0000000..56f6d1a --- /dev/null +++ b/server/internal/storage/sqlstore/federation.go @@ -0,0 +1,323 @@ +package sqlstore + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Federation is the shared SQL implementation of storage.FederationStore. +type Federation struct { + db *sql.DB + rebind func(string) string +} + +// NewFederation builds a federation store. +func NewFederation(db *sql.DB, rebind func(string) string) *Federation { + if rebind == nil { + rebind = func(query string) string { return query } + } + return &Federation{db: db, rebind: rebind} +} + +// The sequence table of this component, and its one counter. +const ( + federationSequences = "federation_sequences" + seqQueue = "queue" +) + +func (s *Federation) q(query string) string { return s.rebind(query) } + +func (s *Federation) tx(ctx context.Context, fn func(*sql.Tx) error) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin transaction: %w", err) + } + defer func() { _ = tx.Rollback() }() + if err := fn(tx); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit transaction: %w", err) + } + return nil +} + +const upsertServerKeySQL = ` + INSERT INTO federation_server_keys + (server_name, key_id, public_key, valid_until_ts, expired_ts, fetched_at_ms) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (server_name, key_id) DO UPDATE SET + public_key = excluded.public_key, + valid_until_ts = excluded.valid_until_ts, + expired_ts = excluded.expired_ts, + fetched_at_ms = excluded.fetched_at_ms` + +// StoreServerKeys implements storage.FederationStore. +func (s *Federation) StoreServerKeys(ctx context.Context, keys []storage.ServerKey) error { + if len(keys) == 0 { + return nil + } + return s.tx(ctx, func(tx *sql.Tx) error { + for _, k := range keys { + if _, err := tx.ExecContext(ctx, s.q(upsertServerKeySQL), + k.ServerName, k.KeyID, k.PublicKey, k.ValidUntilTS, k.ExpiredTS, k.FetchedAtMS); err != nil { + return fmt.Errorf("store key %s of %s: %w", k.KeyID, k.ServerName, err) + } + } + return nil + }) +} + +// ServerKeys implements storage.FederationStore. +func (s *Federation) ServerKeys(ctx context.Context, serverName string, keyIDs []string) ([]storage.ServerKey, error) { + query := ` + SELECT server_name, key_id, public_key, valid_until_ts, expired_ts, fetched_at_ms + FROM federation_server_keys WHERE server_name = ?` + args := []any{serverName} + if len(keyIDs) > 0 { + query += ` AND key_id IN (` + placeholders(len(keyIDs)) + `)` + for _, id := range keyIDs { + args = append(args, id) + } + } + query += ` ORDER BY key_id` + + rows, err := s.db.QueryContext(ctx, s.q(query), args...) + if err != nil { + return nil, fmt.Errorf("read the keys of %q: %w", serverName, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.ServerKey + for rows.Next() { + var k storage.ServerKey + if err := rows.Scan(&k.ServerName, &k.KeyID, &k.PublicKey, + &k.ValidUntilTS, &k.ExpiredTS, &k.FetchedAtMS); err != nil { + return nil, fmt.Errorf("read the keys of %q: %w", serverName, err) + } + out = append(out, k) + } + return out, rows.Err() +} + +const insertQueuedSQL = ` + INSERT INTO federation_queue + (queue_id, destination, kind, room_id, event_id, payload, queued_at_ms) + VALUES (?, ?, ?, ?, ?, ?, ?)` + +// Enqueue implements storage.FederationStore. +func (s *Federation) Enqueue(ctx context.Context, items []storage.QueuedItem) ([]storage.QueuedItem, error) { + if len(items) == 0 { + return nil, nil + } + out := make([]storage.QueuedItem, len(items)) + copy(out, items) + err := s.tx(ctx, func(tx *sql.Tx) error { + top, err := advanceSequence(ctx, tx, s.q, federationSequences, seqQueue, int64(len(out))) + if err != nil { + return err + } + first := top - int64(len(out)) + 1 + for i := range out { + out[i].ID = first + int64(i) + item := out[i] + if _, err := tx.ExecContext(ctx, s.q(insertQueuedSQL), + item.ID, item.Destination, item.Kind, item.RoomID, item.EventID, + string(item.Payload), item.QueuedAtMS); err != nil { + return fmt.Errorf("queue for %s: %w", item.Destination, err) + } + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("enqueue %d items: %w", len(items), err) + } + return out, nil +} + +const selectQueuedSQL = ` + SELECT queue_id, destination, kind, room_id, event_id, payload, queued_at_ms + FROM federation_queue + WHERE destination = ? AND kind = ? + ORDER BY queue_id + LIMIT ?` + +// Queued implements storage.FederationStore. +func (s *Federation) Queued(ctx context.Context, destination, kind string, limit int) ([]storage.QueuedItem, error) { + if limit <= 0 { + return nil, nil + } + rows, err := s.db.QueryContext(ctx, s.q(selectQueuedSQL), destination, kind, limit) + if err != nil { + return nil, fmt.Errorf("read the queue for %q: %w", destination, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.QueuedItem + for rows.Next() { + var ( + item storage.QueuedItem + payload string + ) + if err := rows.Scan(&item.ID, &item.Destination, &item.Kind, &item.RoomID, + &item.EventID, &payload, &item.QueuedAtMS); err != nil { + return nil, fmt.Errorf("read the queue for %q: %w", destination, err) + } + item.Payload = []byte(payload) + out = append(out, item) + } + return out, rows.Err() +} + +// Dequeue implements storage.FederationStore. +func (s *Federation) Dequeue(ctx context.Context, ids []int64) error { + if len(ids) == 0 { + return nil + } + args := make([]any, len(ids)) + for i, id := range ids { + args[i] = id + } + query := `DELETE FROM federation_queue WHERE queue_id IN (` + placeholders(len(ids)) + `)` + if _, err := s.db.ExecContext(ctx, s.q(query), args...); err != nil { + return fmt.Errorf("dequeue %d items: %w", len(ids), err) + } + return nil +} + +const selectQueuedDestinationsSQL = ` + SELECT DISTINCT destination FROM federation_queue ORDER BY destination` + +// QueuedDestinations implements storage.FederationStore. +func (s *Federation) QueuedDestinations(ctx context.Context) ([]string, error) { + rows, err := s.db.QueryContext(ctx, s.q(selectQueuedDestinationsSQL)) + if err != nil { + return nil, fmt.Errorf("read the queued destinations: %w", err) + } + defer func() { _ = rows.Close() }() + + var out []string + for rows.Next() { + var d string + if err := rows.Scan(&d); err != nil { + return nil, fmt.Errorf("read the queued destinations: %w", err) + } + out = append(out, d) + } + return out, rows.Err() +} + +const destinationColumns = ` + server_name, failures, retry_at_ms, last_success_ms, last_failure_ms, last_error` + +func scanDestination(scan func(...any) error) (storage.Destination, error) { + var d storage.Destination + err := scan(&d.ServerName, &d.Failures, &d.RetryAtMS, &d.LastSuccessMS, &d.LastFailureMS, &d.LastError) + return d, err +} + +const selectDestinationSQL = `SELECT ` + destinationColumns + ` + FROM federation_destinations WHERE server_name = ?` + +// Destination implements storage.FederationStore. +func (s *Federation) Destination(ctx context.Context, serverName string) (storage.Destination, error) { + d, err := scanDestination(s.db.QueryRowContext(ctx, s.q(selectDestinationSQL), serverName).Scan) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.Destination{ServerName: serverName}, nil + case err != nil: + return storage.Destination{}, fmt.Errorf("read destination %q: %w", serverName, err) + } + return d, nil +} + +const upsertDestinationSQL = ` + INSERT INTO federation_destinations (` + destinationColumns + `) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (server_name) DO UPDATE SET + failures = excluded.failures, + retry_at_ms = excluded.retry_at_ms, + last_success_ms = excluded.last_success_ms, + last_failure_ms = excluded.last_failure_ms, + last_error = excluded.last_error` + +// SetDestination implements storage.FederationStore. +func (s *Federation) SetDestination(ctx context.Context, d storage.Destination) error { + if _, err := s.db.ExecContext(ctx, s.q(upsertDestinationSQL), + d.ServerName, d.Failures, d.RetryAtMS, d.LastSuccessMS, d.LastFailureMS, d.LastError); err != nil { + return fmt.Errorf("record destination %q: %w", d.ServerName, err) + } + return nil +} + +// selectDestinationSummariesSQL lists every server that either has a +// health row or something queued. The union is what makes a destination +// that has never failed, but has a backlog, show up at all. +const selectDestinationSummariesSQL = ` + SELECT n.server_name, + COALESCE(d.failures, 0), COALESCE(d.retry_at_ms, 0), + COALESCE(d.last_success_ms, 0), COALESCE(d.last_failure_ms, 0), + COALESCE(d.last_error, ''), + (SELECT COUNT(*) FROM federation_queue q WHERE q.destination = n.server_name AND q.kind = 'pdu'), + (SELECT COUNT(*) FROM federation_queue q WHERE q.destination = n.server_name AND q.kind = 'edu') + FROM ( + SELECT server_name FROM federation_destinations + UNION + SELECT DISTINCT destination FROM federation_queue + ) n + LEFT JOIN federation_destinations d ON d.server_name = n.server_name + ORDER BY n.server_name` + +// Destinations implements storage.FederationStore. +func (s *Federation) Destinations(ctx context.Context) ([]storage.DestinationSummary, error) { + rows, err := s.db.QueryContext(ctx, s.q(selectDestinationSummariesSQL)) + if err != nil { + return nil, fmt.Errorf("read the destinations: %w", err) + } + defer func() { _ = rows.Close() }() + + var out []storage.DestinationSummary + for rows.Next() { + var d storage.DestinationSummary + if err := rows.Scan(&d.ServerName, &d.Failures, &d.RetryAtMS, &d.LastSuccessMS, + &d.LastFailureMS, &d.LastError, &d.QueuedPDUs, &d.QueuedEDUs); err != nil { + return nil, fmt.Errorf("read the destinations: %w", err) + } + out = append(out, d) + } + return out, rows.Err() +} + +const selectInboundTxnSQL = ` + SELECT response_json FROM federation_inbound_txns WHERE origin = ? AND txn_id = ?` + +// InboundTransaction implements storage.FederationStore. +func (s *Federation) InboundTransaction(ctx context.Context, origin, txnID string) ([]byte, bool, error) { + var response string + err := s.db.QueryRowContext(ctx, s.q(selectInboundTxnSQL), origin, txnID).Scan(&response) + switch { + case errors.Is(err, sql.ErrNoRows): + return nil, false, nil + case err != nil: + return nil, false, fmt.Errorf("read transaction %q from %q: %w", txnID, origin, err) + } + return []byte(response), true, nil +} + +const insertInboundTxnSQL = ` + INSERT INTO federation_inbound_txns (origin, txn_id, response_json, received_at_ms) + VALUES (?, ?, ?, ?) + ON CONFLICT (origin, txn_id) DO NOTHING` + +// RecordInboundTransaction implements storage.FederationStore. +func (s *Federation) RecordInboundTransaction(ctx context.Context, origin, txnID string, response []byte, atMS int64) error { + if _, err := s.db.ExecContext(ctx, s.q(insertInboundTxnSQL), + origin, txnID, string(response), atMS); err != nil { + return fmt.Errorf("record transaction %q from %q: %w", txnID, origin, err) + } + return nil +} diff --git a/server/internal/storage/sqlstore/rooms.go b/server/internal/storage/sqlstore/rooms.go index 21caa91..1000fba 100644 --- a/server/internal/storage/sqlstore/rooms.go +++ b/server/internal/storage/sqlstore/rooms.go @@ -46,10 +46,14 @@ const ( maxRoomPageSize = 1000 ) -// The two counters in roomserver_sequences. +// The counters in roomserver_sequences. const ( seqStreamOrdering = "stream_ordering" seqStateSnapshot = "state_snapshot" + + // seqBackfillOrdering numbers backfilled history. It starts at 0 and + // only ever goes down; see WriteBackfill. + seqBackfillOrdering = "backfill_ordering" ) // q rebinds a statement for this driver. @@ -121,14 +125,22 @@ func (s *Rooms) Write(ctx context.Context, w storage.RoomWrite) (int64, error) { return err } - snapshotID, err := s.stateSnapshot(ctx, tx, w.Event.RoomID, w.StateBefore) - if err != nil { - return err + // An outlier has no state snapshot: that is what makes it one. + var snapshotID int64 + if !w.Outlier { + if snapshotID, err = s.stateSnapshot(ctx, tx, w.Event.RoomID, w.StateBefore); err != nil { + return err + } } - if err := s.insertEvent(ctx, tx, w.Event, ordering, snapshotID); err != nil { + if err := s.insertEvent(ctx, tx, w.Event, ordering, snapshotID, w.Outlier); err != nil { return err } + if w.Event.Type == memberEventType && w.Event.StateKey != nil { + if err := s.replaceInviteState(ctx, tx, w.Event.RoomID, *w.Event.StateKey, w.InviteState); err != nil { + return err + } + } if w.Redacts != "" { if err := s.applyRedaction(ctx, tx, w.Event.RoomID, w.Redacts, w.Event.EventID, w.RedactedJSON); err != nil { @@ -246,21 +258,55 @@ func stateHash(entries []storage.StateEntry) string { return base64.RawURLEncoding.EncodeToString(h.Sum(nil)) } -const insertEventSQL = ` - INSERT INTO roomserver_events ( - event_id, room_id, stream_ordering, event_type, state_key, - sender, depth, origin_server_ts, event_json, state_before_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` +// memberEventType is the one event type the store has to recognise: its +// rows carry a membership, and its arrival clears a stored invite. +const memberEventType = "m.room.member" -func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEvent, ordering, snapshotID int64) error { - _, err := tx.ExecContext(ctx, s.q(insertEventSQL), +const ( + insertEventSQL = ` + INSERT INTO roomserver_events ( + event_id, room_id, stream_ordering, event_type, state_key, + sender, depth, origin_server_ts, event_json, state_before_id, outlier) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + selectOutlierSQL = `SELECT outlier FROM roomserver_events WHERE event_id = ?` + promoteOutlierSQL = ` + UPDATE roomserver_events + SET stream_ordering = ?, state_before_id = ?, outlier = ? + WHERE event_id = ? AND outlier = ?` +) + +// insertEvent writes the event row. +// +// An event this server already holds is not written twice. The one +// exception is an outlier arriving again as an ordinary event, which is +// promoted in place: it gets the new stream position and the state +// snapshot it now has, and joins the timeline. Its JSON is left as it was — +// the event ID says the two copies are the same event, and the stored one +// may already have been redacted. +func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEvent, ordering, snapshotID int64, outlier bool) error { + var existingOutlier bool + err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), e.EventID).Scan(&existingOutlier) + switch { + case err == nil: + if existingOutlier && !outlier { + if _, err := tx.ExecContext(ctx, s.q(promoteOutlierSQL), + ordering, snapshotID, false, e.EventID, true); err != nil { + return fmt.Errorf("promote outlier: %w", err) + } + } + // The same event twice is not a failure worth propagating: an + // event ID is a hash of its contents, so the second copy says + // exactly what the first one did. + return nil + case !errors.Is(err, sql.ErrNoRows): + return fmt.Errorf("look up event: %w", err) + } + + _, err = tx.ExecContext(ctx, s.q(insertEventSQL), e.EventID, e.RoomID, ordering, e.Type, e.StateKey, - e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID) + e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, outlier) if err != nil { if s.isUnique(err) { - // The same event twice is not a failure worth propagating: an - // event ID is a hash of its contents, so the second copy says - // exactly what the first one did. return nil } return fmt.Errorf("insert event: %w", err) @@ -268,6 +314,147 @@ func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEve return nil } +const ( + deleteInviteStateSQL = `DELETE FROM roomserver_invite_state WHERE room_id = ? AND user_id = ?` + insertInviteStateSQL = ` + INSERT INTO roomserver_invite_state (room_id, user_id, state_json) VALUES (?, ?, ?)` +) + +// replaceInviteState clears a user's stored invite state and, for an +// invite that came with some, stores the new one. +func (s *Rooms) replaceInviteState(ctx context.Context, tx *sql.Tx, roomID, userID string, state []byte) error { + if _, err := tx.ExecContext(ctx, s.q(deleteInviteStateSQL), roomID, userID); err != nil { + return fmt.Errorf("clear invite state: %w", err) + } + if len(state) == 0 { + return nil + } + if _, err := tx.ExecContext(ctx, s.q(insertInviteStateSQL), roomID, userID, string(state)); err != nil { + return fmt.Errorf("store invite state: %w", err) + } + return nil +} + +const selectInviteStateSQL = ` + SELECT state_json FROM roomserver_invite_state WHERE room_id = ? AND user_id = ?` + +// InviteState implements storage.RoomStore. +func (s *Rooms) InviteState(ctx context.Context, roomID, userID string) ([]byte, error) { + var state string + err := s.db.QueryRowContext(ctx, s.q(selectInviteStateSQL), roomID, userID).Scan(&state) + switch { + case errors.Is(err, sql.ErrNoRows): + return nil, nil + case err != nil: + return nil, fmt.Errorf("read the invite state of %q in %q: %w", userID, roomID, err) + } + return []byte(state), nil +} + +// WriteOutliers implements storage.RoomStore. +// +// The stream positions are taken as one block, in one UPDATE, rather than +// one at a time: a join to a large room hands over tens of thousands of +// events, and a counter bumped per event would be that many statements +// holding the counter's row lock. +func (s *Rooms) WriteOutliers(ctx context.Context, roomID string, events []storage.StoredEvent) error { + if len(events) == 0 { + return nil + } + err := s.tx(ctx, func(tx *sql.Tx) error { + top, err := advanceSequence(ctx, tx, s.q, roomSequences, seqStreamOrdering, int64(len(events))) + if err != nil { + return err + } + first := top - int64(len(events)) + 1 + for i, e := range events { + if e.RoomID == "" { + e.RoomID = roomID + } + if err := s.insertEvent(ctx, tx, e, first+int64(i), 0, true); err != nil { + return fmt.Errorf("outlier %s: %w", e.EventID, err) + } + } + return nil + }) + if err != nil { + return fmt.Errorf("write %d outliers to %q: %w", len(events), roomID, err) + } + return nil +} + +// WriteBackfill implements storage.RoomStore. +// +// The positions come from their own counter, advanced downwards by the +// number of events actually written, in one UPDATE: the events already in +// the timeline are sorted out first, inside the transaction, so that the +// block taken is exactly as big as what goes into it. A hole below the +// room would be harmless — nothing streams through backfilled history, it +// is only ever read by bounds — but there is no reason to leave one. +// +// The batch takes the block oldest first: if the counter stood at -10 and +// three events are written, they become -13, -12 and -11. +func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []storage.BackfillEvent) error { + if len(events) == 0 { + return nil + } + err := s.tx(ctx, func(tx *sql.Tx) error { + // Which of the batch is new, which is an outlier to promote, and + // which is already in the timeline and left alone. + var write []storage.BackfillEvent + promote := map[string]bool{} + for _, b := range events { + var outlier bool + err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), b.Event.EventID).Scan(&outlier) + switch { + case errors.Is(err, sql.ErrNoRows): + write = append(write, b) + case err != nil: + return fmt.Errorf("look up event %s: %w", b.Event.EventID, err) + case outlier: + write = append(write, b) + promote[b.Event.EventID] = true + } + } + if len(write) == 0 { + return nil + } + + bottom, err := advanceSequence(ctx, tx, s.q, roomSequences, seqBackfillOrdering, -int64(len(write))) + if err != nil { + return err + } + for i, b := range write { + e := b.Event + if e.RoomID == "" { + e.RoomID = roomID + } + ordering := bottom + int64(i) + snapshotID, err := s.stateSnapshot(ctx, tx, e.RoomID, b.StateBefore) + if err != nil { + return err + } + if promote[e.EventID] { + if _, err := tx.ExecContext(ctx, s.q(promoteOutlierSQL), + ordering, snapshotID, false, e.EventID, true); err != nil { + return fmt.Errorf("promote outlier %s: %w", e.EventID, err) + } + continue + } + if _, err := tx.ExecContext(ctx, s.q(insertEventSQL), + e.EventID, e.RoomID, ordering, e.Type, e.StateKey, + e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, false); err != nil { + return fmt.Errorf("insert event %s: %w", e.EventID, err) + } + } + return nil + }) + if err != nil { + return fmt.Errorf("backfill %d events into %q: %w", len(events), roomID, err) + } + return nil +} + const applyRedactionSQL = ` UPDATE roomserver_events SET event_json = ?, redacted_by = ? WHERE room_id = ? AND event_id = ? AND redacted_by = ''` @@ -387,7 +574,7 @@ func (s *Rooms) recordTransaction(ctx context.Context, tx *sql.Tx, ref storage.T // scanEvent expects. const eventColumns = ` e.room_id, e.event_id, e.event_type, e.state_key, e.sender, - e.depth, e.origin_server_ts, e.event_json, e.stream_ordering, e.redacted_by` + e.depth, e.origin_server_ts, e.event_json, e.stream_ordering, e.redacted_by, e.outlier` // scanEvent reads one row of eventColumns. func scanEvent(scan func(...any) error) (storage.StoredEvent, error) { @@ -397,7 +584,7 @@ func scanEvent(scan func(...any) error) (storage.StoredEvent, error) { body string ) err := scan(&e.RoomID, &e.EventID, &e.Type, &stateKey, &e.Sender, - &e.Depth, &e.OriginServerTS, &body, &e.StreamOrdering, &e.RedactedBy) + &e.Depth, &e.OriginServerTS, &body, &e.StreamOrdering, &e.RedactedBy, &e.Outlier) if err != nil { return storage.StoredEvent{}, err } @@ -892,6 +1079,7 @@ var purgeStatements = []string{ `DELETE FROM roomserver_extremities WHERE room_id = ?`, `DELETE FROM roomserver_transactions WHERE room_id = ?`, `DELETE FROM roomserver_aliases WHERE room_id = ?`, + `DELETE FROM roomserver_invite_state WHERE room_id = ?`, `DELETE FROM roomserver_events WHERE room_id = ?`, `DELETE FROM roomserver_rooms WHERE room_id = ?`, } @@ -968,3 +1156,42 @@ func expectOneRow(result sql.Result, subject string, notFound error) error { } return nil } + +var selectEventByIDSQL = ` + SELECT ` + eventColumns + ` + FROM roomserver_events e WHERE e.event_id = ?` + +// EventByID implements storage.RoomStore. +func (s *Rooms) EventByID(ctx context.Context, eventID string) (storage.StoredEvent, error) { + e, err := scanEvent(s.db.QueryRowContext(ctx, s.q(selectEventByIDSQL), eventID).Scan) + switch { + case errors.Is(err, sql.ErrNoRows): + return storage.StoredEvent{}, fmt.Errorf("event %q: %w", eventID, storage.ErrNoSuchEvent) + case err != nil: + return storage.StoredEvent{}, fmt.Errorf("read event %q: %w", eventID, err) + } + return e, nil +} + +const selectJoinedMemberOnSQL = ` + SELECT 1 FROM roomserver_current_state + WHERE room_id = ? AND event_type = 'm.room.member' AND membership = 'join' + AND state_key LIKE ? ESCAPE '\' + LIMIT 1` + +// HasJoinedMemberOn implements storage.RoomStore. +// +// A user ID's server is everything after its first colon, and a localpart +// cannot contain one, so "ends with ':' plus the server name" is exactly +// "belongs to that server". +func (s *Rooms) HasJoinedMemberOn(ctx context.Context, roomID, serverName string) (bool, error) { + var one int + err := s.db.QueryRowContext(ctx, s.q(selectJoinedMemberOnSQL), roomID, `%:`+escapeLike(serverName)).Scan(&one) + switch { + case errors.Is(err, sql.ErrNoRows): + return false, nil + case err != nil: + return false, fmt.Errorf("look for members of %q in %q: %w", serverName, roomID, err) + } + return true, nil +} diff --git a/server/internal/storage/sqlstore/sequences.go b/server/internal/storage/sqlstore/sequences.go index 5f0e560..950b26f 100644 --- a/server/internal/storage/sqlstore/sequences.go +++ b/server/internal/storage/sqlstore/sequences.go @@ -44,8 +44,15 @@ type execQuerier interface { // statements are two transactions, and two callers can then be handed the // same value. func nextSequence(ctx context.Context, db execQuerier, q func(string) string, table, name string) (int64, error) { - bump := fmt.Sprintf(`UPDATE %s SET value = value + 1 WHERE name = ?`, table) - if _, err := db.ExecContext(ctx, q(bump), name); err != nil { + return advanceSequence(ctx, db, q, table, name, 1) +} + +// advanceSequence hands out a block of n values and returns the last of +// them; the block is the n values ending there. Like nextSequence it must +// be called inside a transaction. +func advanceSequence(ctx context.Context, db execQuerier, q func(string) string, table, name string, n int64) (int64, error) { + bump := fmt.Sprintf(`UPDATE %s SET value = value + ? WHERE name = ?`, table) + if _, err := db.ExecContext(ctx, q(bump), n, name); err != nil { return 0, fmt.Errorf("advance the %s counter: %w", name, err) } return currentSequence(ctx, db, q, table, name) diff --git a/server/internal/storage/sqlstore/sqlstore.go b/server/internal/storage/sqlstore/sqlstore.go index 3c41ff8..32003ec 100644 --- a/server/internal/storage/sqlstore/sqlstore.go +++ b/server/internal/storage/sqlstore/sqlstore.go @@ -59,6 +59,7 @@ type Store struct { users *Users rooms *Rooms sync *Sync + fed *Federation } // New builds a Store. It does not touch the database. @@ -77,6 +78,7 @@ func New(opts Options) (*Store, error) { users: NewUsers(opts.DB, opts.Rebind, opts.IsUniqueViolation), rooms: NewRooms(opts.DB, opts.Rebind, opts.IsUniqueViolation), sync: NewSync(opts.DB, opts.Rebind, opts.IsUniqueViolation), + fed: NewFederation(opts.DB, opts.Rebind), }, nil } @@ -104,6 +106,9 @@ func (s *Store) Rooms() storage.RoomStore { return s.rooms } // Sync implements storage.Database. func (s *Store) Sync() storage.SyncStore { return s.sync } +// Federation implements storage.Database. +func (s *Store) Federation() storage.FederationStore { return s.fed } + // Migrate implements storage.Database. func (s *Store) Migrate(ctx context.Context) error { ms, err := s.migrations() diff --git a/server/internal/storage/sqlstore/timeline.go b/server/internal/storage/sqlstore/timeline.go index 0bad34f..dbe689d 100644 --- a/server/internal/storage/sqlstore/timeline.go +++ b/server/internal/storage/sqlstore/timeline.go @@ -75,7 +75,7 @@ func (s *Rooms) MembershipsForUser(ctx context.Context, userID string, membershi var selectUserMembershipEventsSQL = ` SELECT ` + eventColumns + ` FROM roomserver_events e - WHERE e.room_id = ? AND e.event_type = 'm.room.member' AND e.state_key = ? + WHERE e.room_id = ? AND e.event_type = 'm.room.member' AND e.state_key = ? AND NOT e.outlier ORDER BY e.stream_ordering` // UserMembershipEvents implements storage.RoomStore. @@ -97,13 +97,17 @@ func (s *Rooms) Timeline(ctx context.Context, req storage.TimelineRequest) ([]st return nil, nil } - conditions := []string{`e.room_id = ?`} + // Outliers have no place in a timeline; see the outlier column. + conditions := []string{`e.room_id = ?`, `NOT e.outlier`} args := []any{req.RoomID} - if req.Low > 0 { + // Zero is "unbounded", not a position, at either end: a bound can be + // negative, inside backfilled history, so "is it set" cannot be + // spelled "is it positive". + if req.Low != 0 { conditions = append(conditions, `e.stream_ordering > ?`) args = append(args, req.Low) } - if req.High > 0 { + if req.High != 0 { conditions = append(conditions, `e.stream_ordering <= ?`) args = append(args, req.High) } @@ -229,13 +233,16 @@ func (s *Rooms) RoomsWithEvents(ctx context.Context, roomIDs []string, from, to query := ` SELECT DISTINCT room_id FROM roomserver_events - WHERE room_id IN (` + placeholders(len(roomIDs)) + `) AND stream_ordering > ?` + WHERE room_id IN (` + placeholders(len(roomIDs)) + `) AND NOT outlier` args := make([]any, 0, len(roomIDs)+2) for _, id := range roomIDs { args = append(args, id) } - args = append(args, from) - if to > 0 { + if from != 0 { + query += ` AND stream_ordering > ?` + args = append(args, from) + } + if to != 0 { query += ` AND stream_ordering <= ?` args = append(args, to) } @@ -261,7 +268,7 @@ func (s *Rooms) RoomsWithEvents(ctx context.Context, roomIDs []string, from, to const selectEventAtStreamSQL = ` SELECT e.event_id, e.event_type, e.state_key, e.state_before_id FROM roomserver_events e - WHERE e.room_id = ? AND e.stream_ordering <= ? + WHERE e.room_id = ? AND NOT e.outlier AND e.stream_ordering <= ? ORDER BY e.stream_ordering DESC LIMIT 1` @@ -278,7 +285,7 @@ const selectSnapshotEntriesSQL = ` // the snapshot is the state the event was authorised against, which is the // state *before* it. func (s *Rooms) StateAtStream(ctx context.Context, roomID string, pos int64) ([]storage.StateEntry, error) { - if pos <= 0 { + if pos == 0 { return nil, nil } diff --git a/server/internal/storage/storage.go b/server/internal/storage/storage.go index 7638f9f..d0f1743 100644 --- a/server/internal/storage/storage.go +++ b/server/internal/storage/storage.go @@ -71,6 +71,10 @@ type Database interface { // uploaded filters. Sync() SyncStore + // Federation is the federation store: other servers' keys, the + // outbound queue and its backoff, and the inbound transaction log. + Federation() FederationStore + // Close releases the connection pool. Close() error } diff --git a/server/internal/storage/sync.go b/server/internal/storage/sync.go index f0d9dda..e735cbc 100644 --- a/server/internal/storage/sync.go +++ b/server/internal/storage/sync.go @@ -59,10 +59,17 @@ const ( // lets a token a client was handed be passed straight back without // re-reading the event it names: a position is somewhere a client has got // to, not an event it still wants. +// +// Positions can be negative: history backfilled from other servers is +// numbered downwards from -1, below everything that was already here (see +// RoomStore.WriteBackfill). Zero is never a position, which is what lets +// it go on meaning "unbounded" here — and why the position just below an +// event is PositionBefore and not a plain subtraction. type TimelineRequest struct { RoomID string - // Low is an exclusive lower bound. Zero means the start of the room. + // Low is an exclusive lower bound. Zero means unbounded: the start of + // the room, backfilled history included. Low int64 // High is an inclusive upper bound. Zero means the top of the stream, @@ -84,6 +91,21 @@ type TimelineRequest struct { Filter TimelineFilter } +// PositionBefore is the position just below pos: the bound that names +// "everything up to, but not including, the event at pos". +// +// It is pos-1 except at 1, whose predecessor is -1. Zero is not a +// position — it is the "unbounded" bound of every window — so a token +// below the first event this server ever persisted has to skip it, or +// handing that token back as a High bound would read the whole room again +// instead of the backfilled history beneath it. +func PositionBefore(pos int64) int64 { + if pos == 1 { + return -1 + } + return pos - 1 +} + // TimelineFilter is the part of a Client-Server API event filter that the // store can apply itself. // diff --git a/server/internal/syncapi/context.go b/server/internal/syncapi/context.go index b424676..c8d606d 100644 --- a/server/internal/syncapi/context.go +++ b/server/internal/syncapi/context.go @@ -106,7 +106,7 @@ func (s *API) Context(ctx context.Context, req ContextRequest) (ContextResponse, User: req.User, // Strictly before the target: High is inclusive, and the // target itself is reported separately. - High: position - 1, + High: storage.PositionBefore(position), Direction: storage.DirectionBackward, Limit: before, Filter: filter.TimelineFilter(), @@ -120,7 +120,7 @@ func (s *API) Context(ctx context.Context, req ContextRequest) (ContextResponse, out.Start = pduToken(earlier.Start).String() } if out.Start == "" { - out.Start = pduToken(position - 1).String() + out.Start = pduToken(storage.PositionBefore(position)).String() } if after > 0 { diff --git a/server/internal/syncapi/messages.go b/server/internal/syncapi/messages.go index 9ee2c07..acea0f8 100644 --- a/server/internal/syncapi/messages.go +++ b/server/internal/syncapi/messages.go @@ -112,6 +112,9 @@ func (s *API) Messages(ctx context.Context, req MessagesRequest) (MessagesRespon Direction: direction, Limit: limit, Filter: filter.TimelineFilter(), + // Scrolling up past the oldest event this server holds of a room + // joined over federation asks the room's other servers for more. + Backfill: true, }) if err != nil { return MessagesResponse{}, err diff --git a/server/internal/syncapi/outbound.go b/server/internal/syncapi/outbound.go new file mode 100644 index 0000000..35a66be --- /dev/null +++ b/server/internal/syncapi/outbound.go @@ -0,0 +1,31 @@ +package syncapi + +import ( + "context" + "encoding/json" + "time" +) + +// Outbound passes this server's ephemeral events on to other servers. +// +// Typing notifications, read receipts and to-device messages are not room +// events and never go through the roomserver, so the federation queue +// cannot pick them up from there; the sync API, which is where they are +// written, hands them over instead. Only what a local user did is handed +// over — the same calls are how an EDU from another server is taken in, +// and passing those on again would echo them back to where they came +// from. +// +// Every method must return quickly: they are called on the request path +// of the client that typed, read or sent. +type Outbound interface { + Typing(ctx context.Context, roomID, userID string, typing bool, timeout time.Duration) + Receipt(ctx context.Context, roomID, userID, receiptType, eventID, threadID string, ts int64) + ToDevice(ctx context.Context, sender, eventType string, messages map[string]map[string]json.RawMessage) +} + +// SetOutbound connects the federation queue. It is a setter rather than an +// option because the queue is built after the sync API's dependencies and +// before the sync API is serving, and nil — no federation — is a valid +// value throughout. +func (s *API) SetOutbound(o Outbound) { s.outbound = o } diff --git a/server/internal/syncapi/presence.go b/server/internal/syncapi/presence.go index d316c3d..54518a5 100644 --- a/server/internal/syncapi/presence.go +++ b/server/internal/syncapi/presence.go @@ -104,7 +104,7 @@ func (s *API) Presence(ctx context.Context, userID string) (storage.PresenceStat return storage.PresenceState{}, ErrPresenceDisabled } if !s.isLocal(userID) { - // Remote presence needs federation (Phase 5). Reporting offline + // Remote presence is not fetched over federation. Reporting offline // would be a guess; saying so is not. return storage.PresenceState{}, errForbiddenf( "%s belongs to another server and federation is required to ask it", userID) diff --git a/server/internal/syncapi/receipts.go b/server/internal/syncapi/receipts.go index aaef608..4a01c8b 100644 --- a/server/internal/syncapi/receipts.go +++ b/server/internal/syncapi/receipts.go @@ -104,6 +104,10 @@ func (s *API) SetReceipt(ctx context.Context, req ReceiptRequest) error { return nil } s.notifier.NotifyRoom(req.RoomID) + // A private receipt never leaves the server, which is the point of it. + if s.outbound != nil && s.isLocal(req.User) { + s.outbound.Receipt(ctx, req.RoomID, req.User, req.ReceiptType, req.EventID, req.ThreadID, s.nowMS()) + } return nil } diff --git a/server/internal/syncapi/syncapi.go b/server/internal/syncapi/syncapi.go index 08ec4e6..0475a49 100644 --- a/server/internal/syncapi/syncapi.go +++ b/server/internal/syncapi/syncapi.go @@ -90,6 +90,10 @@ type API struct { // asking, where a clear refusal tells it this one has it turned off. presenceEnabled bool + // outbound passes local typing, receipts and remote to-device + // messages to the federation queue; nil without federation. + outbound Outbound + // longPollTimeout caps how long a /sync may block, whatever ?timeout= // the client asked for. A client asking for ten minutes would hold a // connection and a database handle for ten minutes. diff --git a/server/internal/syncapi/todevice.go b/server/internal/syncapi/todevice.go index d84e1b2..62f707d 100644 --- a/server/internal/syncapi/todevice.go +++ b/server/internal/syncapi/todevice.go @@ -33,13 +33,13 @@ type SendToDeviceRequest struct { // SendToDevice queues messages for delivery. // -// A message to a user of another server is dropped rather than refused. -// That is deliberate: a client encrypting for a room with remote members -// sends one message per device in one request, and failing the request -// because federation is off would stop it from delivering to the local -// devices as well. The dropped recipients simply never see the key and -// their client reports an undecryptable message, which is what they would -// see anyway. Phase 5 routes them. +// A message to a user of another server goes to the federation queue. With +// federation off it is dropped rather than refused. That is deliberate: a +// client encrypting for a room with remote members sends one message per +// device in one request, and failing the request because federation is +// off would stop it from delivering to the local devices as well. The +// dropped recipients simply never see the key and their client reports an +// undecryptable message, which is what they would see anyway. func (s *API) SendToDevice(ctx context.Context, req SendToDeviceRequest) error { if req.EventType == "" { return errInvalidParamf("an event type is required") @@ -48,11 +48,19 @@ func (s *API) SendToDevice(ctx context.Context, req SendToDeviceRequest) error { var ( queued []storage.NewToDeviceMessage woken []string + remote = map[string]map[string]json.RawMessage{} sentAt = s.nowMS() ) for userID, byDevice := range req.Messages { localpart, ok := s.localpartOf(userID) if !ok { + // A remote recipient goes to the federation queue when there + // is one and the sender is ours; a remote sender's message to + // a remote user is not this server's to forward. + if s.outbound != nil && s.isLocal(req.Sender) { + remote[userID] = byDevice + continue + } s.log.Debug("dropping a to-device message for a remote user", "sender", req.Sender, "recipient", userID) continue @@ -95,6 +103,9 @@ func (s *API) SendToDevice(ctx context.Context, req SendToDeviceRequest) error { woken = append(woken, userID) } + if len(remote) > 0 { + s.outbound.ToDevice(ctx, req.Sender, req.EventType, remote) + } if len(queued) == 0 { return nil } diff --git a/server/internal/syncapi/token.go b/server/internal/syncapi/token.go index 4d5a1b1..d493449 100644 --- a/server/internal/syncapi/token.go +++ b/server/internal/syncapi/token.go @@ -96,7 +96,10 @@ func ParseToken(raw string) (Token, error) { var positions [5]int64 for i, field := range fields { value, err := strconv.ParseInt(field, 10, 64) - if err != nil || value < 0 { + // The PDU position alone may be negative: /messages hands out + // positions inside history backfilled from other servers, which + // is numbered downwards from -1. Every other stream starts at 0. + if err != nil || (value < 0 && i > 0) { return Token{}, fmt.Errorf("%q is not a sync token: position %d is %q", raw, i+1, field) } positions[i] = value diff --git a/server/internal/syncapi/token_test.go b/server/internal/syncapi/token_test.go index 88dfd38..1dba487 100644 --- a/server/internal/syncapi/token_test.go +++ b/server/internal/syncapi/token_test.go @@ -20,6 +20,13 @@ func TestTokenRoundTrips(t *testing.T) { syncapi.Token{PDU: 1, Receipt: 2, ToDevice: 3, AccountData: 4, Presence: 5}, "s1_2_3_4_5", }, + { + // /messages pages back into history backfilled from other + // servers, which is numbered downwards from -1. + "a position inside backfilled history", + syncapi.Token{PDU: -7}, + "s-7", + }, { // A zero in the middle has to be written, because the // positions are read by place. @@ -77,7 +84,7 @@ func TestParseTokenRejectsNonsense(t *testing.T) { "nonsense", // no prefix "t1_2", // a prefix this build does not write "s", // a prefix and nothing else - "s-1", // a negative position + "s1_-1", // a negative position outside the event stream "sx", // not a number "s1_", // a trailing separator "s1_2_3_4_5_6", // more positions than there are streams diff --git a/server/internal/syncapi/typing.go b/server/internal/syncapi/typing.go index 2ff3b55..98564e1 100644 --- a/server/internal/syncapi/typing.go +++ b/server/internal/syncapi/typing.go @@ -196,6 +196,9 @@ func (s *API) SetTyping(ctx context.Context, req TypingRequest) error { } s.typing.set(req.RoomID, req.User, req.Typing, req.Timeout) + if s.outbound != nil && s.isLocal(req.User) { + s.outbound.Typing(ctx, req.RoomID, req.User, req.Typing, req.Timeout) + } return nil } diff --git a/server/test/complement/Dockerfile b/server/test/complement/Dockerfile index 2ac0208..9b0cfe7 100644 --- a/server/test/complement/Dockerfile +++ b/server/test/complement/Dockerfile @@ -16,10 +16,10 @@ # # docker build -t complement-warp -f server/test/complement/Dockerfile . # -# Federation (port 8448, served over TLS with a certificate signed by the CA -# Complement mounts at /complement/ca) arrives in Phase 5. Until then this -# image is for the federation=off profile: the C-S tests. The port is still -# exposed, because Complement expects it to be. +# Federation is served on 8448 over TLS, with a certificate the entrypoint +# has the CA Complement mounts at /complement/ca sign. The federation +# profile of run.sh switches it on; the default profile leaves it off and +# runs the C-S tests only. FROM golang:1.26-alpine AS build @@ -38,9 +38,9 @@ RUN go build -trimpath -o /out/warp ./cmd/warp FROM alpine:3.21 -# curl is what the health check uses; ca-certificates is for the outbound -# federation requests of Phase 5. -RUN apk add --no-cache ca-certificates curl +# curl is what the health check uses, ca-certificates the roots outbound +# federation trusts, and openssl signs the federation certificate. +RUN apk add --no-cache ca-certificates curl openssl COPY --from=build /out/warp /usr/local/bin/warp COPY server/test/complement/entrypoint.sh /usr/local/bin/warp-complement diff --git a/server/test/complement/README.md b/server/test/complement/README.md index 660f64e..bd28036 100644 --- a/server/test/complement/README.md +++ b/server/test/complement/README.md @@ -6,7 +6,8 @@ Matrix integration suite and this project's primary progress metric drives it over the real HTTP API. ```sh -make complement # the current target set +make complement # the C-S set, federation off +make complement-federation # the federation set bash run.sh TestLogin # one test COMPLEMENT_DEBUG=1 make complement # with the container logs ``` @@ -25,12 +26,26 @@ the phases that are done. ## Profiles -**`federation=off`** — what runs today. Only the Client-Server tests; -`WARP_FEDERATION_ENABLED=false`, port 8448 exposed but unserved. +`COMPLEMENT_PROFILE` picks one; each has its own test set in `run.sh`. -**`federation=on`** — Phase 5. It needs a TLS listener on 8448 with a -certificate signed by the CA that Complement mounts at `/complement/ca`, -which is the first thing to add here when federation lands. +**`csapi`** (the default, `make complement`) — the Client-Server tests in +`tests/csapi`, with `WARP_FEDERATION_ENABLED=false`. Port 8448 is still +served over TLS, because the CA is mounted either way, but the gate answers +every federation request with `M_UNRECOGNIZED`. + +**`federation`** (`make complement-federation`) — the federation tests in +`tests/` and the multi-server tests in `tests/csapi`, with federation on. +The entrypoint has Complement's CA at `/complement/ca` sign a certificate +for the server's name, serves the API on 8448 with it, and points +`federation.tls_ca_file` at the same CA, so that Warp trusts the other +homeservers of the run and the servers Complement itself pretends to be. +The switch reaches the container through `COMPLEMENT_SHARE_ENV_PREFIX`: +`run.sh` sets `PASS_WARP_FEDERATION_ENABLED`, which Complement passes on +as `WARP_FEDERATION_ENABLED`. + +The federation set leaves out partial-state joins (MSC3706, +`TestPartialStateJoin*`), which Warp does not implement, and anything that +needs application services (Phase 9) or device lists (Phase 7). ## The test set diff --git a/server/test/complement/entrypoint.sh b/server/test/complement/entrypoint.sh index a33e0bc..de6656c 100755 --- a/server/test/complement/entrypoint.sh +++ b/server/test/complement/entrypoint.sh @@ -29,11 +29,35 @@ export WARP_RATE_LIMIT_ENABLED=false # Open registration: the tests create their own users. export WARP_REGISTRATION_ENABLED=true -# Federation stays off unless the profile asks for it. Phase 5 adds the -# TLS listener on 8448 that the federation profile needs; until then this -# image only serves the Client-Server API. +# Federation stays off unless the profile asks for it: run.sh passes +# WARP_FEDERATION_ENABLED=true into the container for the federation +# profile. export WARP_FEDERATION_ENABLED="${WARP_FEDERATION_ENABLED:-false}" +# Federation over TLS on 8448. Complement mounts a CA at /complement/ca and +# expects every homeserver of a run to present a certificate it signed; +# the same CA is what Warp trusts when it connects to the others, and to +# the servers Complement itself pretends to be. +CA_DIR=/complement/ca +if [ -f "$CA_DIR/ca.crt" ] && [ -f "$CA_DIR/ca.key" ]; then + if [ ! -f "$DATA_DIR/tls.crt" ]; then + openssl genrsa -out "$DATA_DIR/tls.key" 2048 2>/dev/null + openssl req -new -sha256 -key "$DATA_DIR/tls.key" \ + -subj "/CN=$WARP_SERVER_NAME" -out "$DATA_DIR/tls.csr" + printf 'subjectAltName=DNS:%s\n' "$WARP_SERVER_NAME" > "$DATA_DIR/san.ext" + # The serial file goes in the data directory: the CA mount may be + # read-only. + openssl x509 -req -in "$DATA_DIR/tls.csr" \ + -CA "$CA_DIR/ca.crt" -CAkey "$CA_DIR/ca.key" \ + -CAserial "$DATA_DIR/ca.srl" -CAcreateserial \ + -out "$DATA_DIR/tls.crt" -days 1 -sha256 -extfile "$DATA_DIR/san.ext" 2>/dev/null + fi + export WARP_LISTEN_FEDERATION_ADDRESS=":8448" + export WARP_TLS_CERT_FILE="$DATA_DIR/tls.crt" + export WARP_TLS_KEY_FILE="$DATA_DIR/tls.key" + export WARP_FEDERATION_TLS_CA_FILE="$CA_DIR/ca.crt" +fi + # Complement captures the container's output, so text at debug level is the # most useful thing to have when a test fails. export WARP_LOG_LEVEL="${WARP_LOG_LEVEL:-debug}" diff --git a/server/test/complement/run.sh b/server/test/complement/run.sh index f072711..24ddde3 100755 --- a/server/test/complement/run.sh +++ b/server/test/complement/run.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # Run Complement against a freshly built Warp image. # -# server/test/complement/run.sh # the test set for the phases that are done -# server/test/complement/run.sh TestLogin # one test +# server/test/complement/run.sh # the C-S test set, federation off +# COMPLEMENT_PROFILE=federation server/test/complement/run.sh # the federation test set +# server/test/complement/run.sh TestLogin # one test # COMPLEMENT_TESTS='Test.*' server/test/complement/run.sh # # Complement is a separate Go module, so it has to be checked out and run on @@ -16,17 +17,42 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" : "${COMPLEMENT_REF:=main}" : "${COMPLEMENT_DIR:=${REPO_ROOT}/.complement}" : "${COMPLEMENT_IMAGE:=complement-warp}" +: "${COMPLEMENT_PROFILE:=csapi}" -# The target set for the phases that are done, from PLAN.md section 7. Add -# to it as phases land rather than switching to the whole suite: a pass +# The target sets for the phases that are done, from PLAN.md section 7. Add +# to them as phases land rather than switching to the whole suite: a pass # count that goes down because a later phase is missing is noise. # -# Phase 4 adds the stream tests: /sync in all its forms, /messages, -# /context, filters, typing, receipts and to-device messaging. What is still -# out is anything needing a later phase — the notification counts of the -# push rules (Phase 8), the device lists of the keyserver (Phase 7), and -# the whole of federation (Phase 5). -: "${COMPLEMENT_TESTS:=TestRegistration|TestLogin|TestDeviceManagement|TestRoomCreate|TestRoomState|TestRoomAlias|TestRoomForget|TestMembership.*|TestRedaction.*|TestJoinRoom.*|TestLeaveRoom.*|TestKickRoom.*|TestBanRoom.*|TestPublicRoom.*|TestSync.*|TestMessages.*|TestContext.*|TestFilter.*|TestTyping.*|TestReceipt.*|TestToDevice.*|TestRoomCreationReportsEventsToMyself|TestLazyLoading.*|TestWriteMDirectAccountData}" +# csapi: the Client-Server tests, with federation off. Phase 4 added the +# stream tests: /sync in all its forms, /messages, /context, filters, +# typing, receipts and to-device messaging. What is still out is anything +# needing a later phase — the notification counts of the push rules +# (Phase 8) and the device lists of the keyserver (Phase 7). +CSAPI_TESTS='TestRegistration|TestLogin|TestDeviceManagement|TestRoomCreate|TestRoomState|TestRoomAlias|TestRoomForget|TestMembership.*|TestRedaction.*|TestJoinRoom.*|TestLeaveRoom.*|TestKickRoom.*|TestBanRoom.*|TestPublicRoom.*|TestSync.*|TestMessages.*|TestContext.*|TestFilter.*|TestTyping.*|TestReceipt.*|TestToDevice.*|TestRoomCreationReportsEventsToMyself|TestLazyLoading.*|TestWriteMDirectAccountData' + +# federation: Phase 5. Joins, leaves and invites across servers, keys, +# transactions, missing events, server ACLs and the queries. Partial-state +# joins (MSC3706, TestPartialStateJoin*) are not implemented and are left +# out; so is anything needing an application service (Phase 9) or device +# lists (Phase 7). +FEDERATION_TESTS='TestFederation.*|TestJoinViaRoomIDAndServerName|TestJoinFederatedRoom.*|TestBannedUserCannotSendJoin|TestCannotSendNon.*Via.*|TestCannotSendKnockViaSend.*|TestOutboundFederation.*|TestInboundFederation.*|TestInboundCanReturnMissingEvents|TestRemoteAliasRequestsUnderstandUnicode|TestNetworkPartitionOrdering|TestACLs|TestGetMissingEventsGapFilling|TestEventAuth|TestUnbanViaInvite|TestJoinRoomWithFederation.*|TestRemote.*' + +case "$COMPLEMENT_PROFILE" in + csapi) + : "${COMPLEMENT_TESTS:=$CSAPI_TESTS}" + PACKAGES=(./tests/csapi/...) + FEDERATION_ENABLED=false + ;; + federation) + : "${COMPLEMENT_TESTS:=$FEDERATION_TESTS}" + PACKAGES=(./tests/... ) + FEDERATION_ENABLED=true + ;; + *) + echo "complement: unknown COMPLEMENT_PROFILE '$COMPLEMENT_PROFILE' (want csapi or federation)" >&2 + exit 2 + ;; +esac if [[ $# -gt 0 ]]; then COMPLEMENT_TESTS="$(IFS='|'; echo "$*")" @@ -49,12 +75,18 @@ else git -C "$COMPLEMENT_DIR" checkout -q FETCH_HEAD fi -echo "complement: running '${COMPLEMENT_TESTS}' with federation off" +echo "complement: running '${COMPLEMENT_TESTS}' (profile ${COMPLEMENT_PROFILE}, federation ${FEDERATION_ENABLED})" cd "$COMPLEMENT_DIR" # COMPLEMENT_SPAWN_HS_TIMEOUT_SECS is generous because the first container # of a run has to apply the migrations on a cold SQLite file. +# +# COMPLEMENT_SHARE_ENV_PREFIX hands every host variable starting with +# PASS_ to the homeserver containers, prefix removed: it is how the profile +# reaches the entrypoint's federation switch. COMPLEMENT_BASE_IMAGE="$COMPLEMENT_IMAGE" \ COMPLEMENT_SPAWN_HS_TIMEOUT_SECS="${COMPLEMENT_SPAWN_HS_TIMEOUT_SECS:-60}" \ COMPLEMENT_DEBUG="${COMPLEMENT_DEBUG:-0}" \ - go test -count=1 -v -timeout 20m -run "^(${COMPLEMENT_TESTS})$" ./tests/csapi/... + COMPLEMENT_SHARE_ENV_PREFIX=PASS_ \ + PASS_WARP_FEDERATION_ENABLED="$FEDERATION_ENABLED" \ + go test -count=1 -v -timeout 30m -run "^(${COMPLEMENT_TESTS})$" "${PACKAGES[@]}" diff --git a/warp.yaml.example b/warp.yaml.example index abd7826..95f2ece 100644 --- a/warp.yaml.example +++ b/warp.yaml.example @@ -40,6 +40,15 @@ listen: max_header_bytes: 1048576 # WARP_MAX_HEADER_BYTES shutdown_timeout: 30s # WARP_SHUTDOWN_TIMEOUT + # A second listener serving the same API over TLS, for other servers. + # 8448 is the port they try when nothing delegates them elsewhere. Leave + # it empty behind a reverse proxy that terminates TLS, and point + # global.well_known.server_address at the proxy instead. + # WARP_LISTEN_FEDERATION_ADDRESS + federation_address: "" + tls_cert_file: "" # WARP_TLS_CERT_FILE (PEM chain) + tls_key_file: "" # WARP_TLS_KEY_FILE + database: # sqlite (dev, single user), postgres (production) or pocketbase # (appliance; not implemented yet). WARP_DB_DRIVER @@ -156,6 +165,19 @@ federation: # WARP_FEDERATION_ALLOW_RUNTIME_TOGGLE allow_runtime_toggle: true + # Notaries asked for another server's signing key when that server + # cannot be reached directly, e.g. [matrix.org]. Empty: keys are only + # fetched from their own server. WARP_FEDERATION_TRUSTED_KEY_SERVERS + trusted_key_servers: [] + + # How long other servers may cache this server's key. At most 168h. + # WARP_FEDERATION_KEY_VALIDITY_PERIOD + key_validity_period: 24h + + # Extra CA certificates (PEM) trusted for outbound federation, for a + # private federation with an internal CA. WARP_FEDERATION_TLS_CA_FILE + tls_ca_file: "" + registration: # Off by default. Make the first account with # `warp create-account --username you --admin`, then turn registration