From b50775317230ec8d13b89d36196446be7d777713 Mon Sep 17 00:00:00 2001 From: tajniak81 <13187254+tajniak81@users.noreply.github.com> Date: Sat, 17 Oct 2026 11:43:36 +0200 Subject: [PATCH] Phase 8: push and notifications Implements PLAN.md Phase 8. internal/pushrules is the rule language: the spec's default ruleset, all six condition kinds, glob and word-boundary matching, intentional mentions switching off the legacy body rules, and rule ordering as Synapse has it. internal/pushserver keeps the rules, pushers and notifications and talks to push gateways. Push rules are the m.push_rules account data, written only by the push server; reads keep the user's own rules and the enabled/actions of each default and take the rest from the current defaults, so new default rules reach every user without a migration. PUT account_data/m.push_rules is refused with 405. The push server listens on the roomserver's emit stage and evaluates each event on its own worker for every local member and an invite's target. Notifications and per-room unread counts are stored in migration 0010; counts are a seventh sync token stream, so /sync reports unread_notifications and picks up count changes made after the event or by a receipt from another device. m.read and m.read.private receipts clear the counts; leaving a room clears it. Pushers (http kind only) push through a bounded sender pool with retries, honour event_id_only, drop push keys the gateway rejects, and go away with their device. GET /notifications pages the history. New config section push: enabled, include_content, gateway_timeout, notification_retention. Account data writes now wake the user's blocked syncs. Co-Authored-By: Claude Opus 5.5 --- README.md | 21 +- docs/README.md | 1 + docs/configuration.md | 4 + docs/push.md | 235 ++++++++++ docs/sync.md | 24 +- server/internal/clientapi/accountdata.go | 19 +- server/internal/clientapi/clientapi.go | 8 + server/internal/clientapi/push.go | 303 +++++++++++++ server/internal/clientapi/sync_test.go | 2 +- server/internal/config/config.go | 36 ++ server/internal/pushrules/actions.go | 160 +++++++ server/internal/pushrules/defaults.go | 132 ++++++ server/internal/pushrules/edit.go | 107 +++++ server/internal/pushrules/evaluate.go | 360 +++++++++++++++ server/internal/pushrules/pushrules_test.go | 388 ++++++++++++++++ server/internal/pushrules/rules.go | 273 ++++++++++++ server/internal/pushserver/gateway.go | 243 ++++++++++ server/internal/pushserver/gateway_test.go | 109 +++++ server/internal/pushserver/notifications.go | 162 +++++++ server/internal/pushserver/pushers.go | 179 ++++++++ server/internal/pushserver/pushserver.go | 251 +++++++++++ server/internal/pushserver/rules.go | 167 +++++++ server/internal/pushserver/worker.go | 278 ++++++++++++ server/internal/roomserver/push.go | 97 ++++ server/internal/setup/federation.go | 18 + server/internal/setup/phase8_test.go | 415 ++++++++++++++++++ server/internal/setup/routes.go | 12 +- server/internal/setup/server.go | 39 +- .../internal/storage/pocketbase/pocketbase.go | 50 +++ .../storage/postgres/migrations/0010_push.sql | 70 +++ server/internal/storage/push.go | 130 ++++++ server/internal/storage/push_test.go | 218 +++++++++ .../storage/sqlite/migrations/0010_push.sql | 70 +++ server/internal/storage/sqlstore/push.go | 372 ++++++++++++++++ server/internal/storage/sqlstore/sqlstore.go | 5 + server/internal/storage/sqlstore/sync.go | 5 + server/internal/storage/storage.go | 4 + server/internal/storage/sync.go | 4 + server/internal/syncapi/notifications.go | 51 +++ server/internal/syncapi/receipts.go | 5 + server/internal/syncapi/sync.go | 55 ++- server/internal/syncapi/syncapi.go | 22 +- server/internal/syncapi/token.go | 20 +- server/internal/syncapi/token_test.go | 20 +- server/internal/userapi/userapi.go | 5 +- server/test/complement/run.sh | 16 +- warp.yaml.example | 21 + 47 files changed, 5136 insertions(+), 50 deletions(-) create mode 100644 docs/push.md create mode 100644 server/internal/clientapi/push.go create mode 100644 server/internal/pushrules/actions.go create mode 100644 server/internal/pushrules/defaults.go create mode 100644 server/internal/pushrules/edit.go create mode 100644 server/internal/pushrules/evaluate.go create mode 100644 server/internal/pushrules/pushrules_test.go create mode 100644 server/internal/pushrules/rules.go create mode 100644 server/internal/pushserver/gateway.go create mode 100644 server/internal/pushserver/gateway_test.go create mode 100644 server/internal/pushserver/notifications.go create mode 100644 server/internal/pushserver/pushers.go create mode 100644 server/internal/pushserver/pushserver.go create mode 100644 server/internal/pushserver/rules.go create mode 100644 server/internal/pushserver/worker.go create mode 100644 server/internal/roomserver/push.go create mode 100644 server/internal/setup/phase8_test.go create mode 100644 server/internal/storage/postgres/migrations/0010_push.sql create mode 100644 server/internal/storage/push.go create mode 100644 server/internal/storage/push_test.go create mode 100644 server/internal/storage/sqlite/migrations/0010_push.sql create mode 100644 server/internal/storage/sqlstore/push.go create mode 100644 server/internal/syncapi/notifications.go diff --git a/README.md b/README.md index f417697..4beb2f7 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,21 @@ what exists today. ## Status +**Phase 8 — push and notifications. Complete.** Every event is checked +against the push rules of every local member of its room. Warp has the +spec's full default ruleset, all six condition kinds, intentional mentions +switching off the older body-matching rules, and user rules ordered the way +Synapse orders them. Push rules live in `m.push_rules` account data, so +clients find them where they already look, while a server upgrade still +brings in new default rules. `/sync` reports `unread_notifications` for +each joined room. The counts are the seventh stream of a sync token, so a +receipt sent from another device clears them everywhere at once. Pushers +register an app's push gateway; pushes go out from a bounded pool with +retries, the `event_id_only` format is honoured, and push keys the gateway +rejects are removed. Logging a device out removes its pushers. Known gaps — +no thread counts, no email pushers — are listed in +[docs/push.md](docs/push.md). + **Phase 7 — end-to-end encryption. Complete.** The server half of E2EE: devices upload their identity, one-time and fallback keys, other devices query and claim them — across servers too — and every one-time key is @@ -118,6 +133,9 @@ Client-Server API: | `POST /keys/upload`, `/keys/query`, `/keys/claim` · `GET /keys/changes` | device, one-time and fallback keys, local and from other servers | | `POST /keys/device_signing/upload`, `/keys/signatures/upload` | cross-signing, with the password re-asked to replace a master key | | `/room_keys/version[/{v}]` · `/room_keys/keys[/{room}[/{session}]]` | server-side key backup; see [docs/e2ee.md](docs/e2ee.md) | +| `GET /pushrules/` · `GET`/`PUT`/`DELETE /pushrules/global/{kind}/{rule}` and its `enabled` / `actions` | push rules, over the spec's defaults; see [docs/push.md](docs/push.md) | +| `GET /pushers` · `POST /pushers/set` | where a device wants its notifications pushed | +| `GET /notifications` | what notified you, newest first | | `GET`/`PUT /presence/{user}/status` | presence — **only while `sync.presence` is on**, and it is off by default | | `POST /_matrix/media/v3/upload` · `POST /_matrix/media/v1/create` · `PUT /_matrix/media/v3/upload/{server}/{id}` | uploads, directly or to a reserved URI | | `GET /_matrix/client/v1/media/download/…` · `/thumbnail/…` · `/config` | downloads and thumbnails, local and from other servers; see [docs/media.md](docs/media.md) | @@ -144,7 +162,8 @@ profiles. `make complement` runs the account, room and stream tests with federation off — `TestRegistration`, `TestLogin`, `TestDeviceManagement`, the Phase 3 room tests and the Phase 4 `TestSync*`, `TestMessages*`, `TestContext*`, `TestFilter*`, `TestTyping*`, `TestReceipt*` and -`TestToDevice*`, the Phase 6 media tests and the Phase 7 key tests. +`TestToDevice*`, the Phase 6 media tests, the Phase 7 key tests and the +Phase 8 push rule tests. `make complement-federation` runs the Phase 5 Server-Server tests and the device list tests with federation on, over TLS with Complement's CA. diff --git a/docs/README.md b/docs/README.md index db6366f..547a7bb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,7 @@ | [federation.md](federation.md) | the Server-Server API: discovery, keys, taking in events, joins across servers, the outbound queue, the switch | | [media.md](media.md) | `internal/mediaapi`: uploads, downloads, thumbnails, other servers' media, where the files go | | [e2ee.md](e2ee.md) | `internal/keyserver`: device and one-time keys, cross-signing, device list tracking, key backup | +| [push.md](push.md) | `internal/pushrules` and `internal/pushserver`: push rules, notification counts, pushers, the push gateway client | | [admin-api.md](admin-api.md) | `/_warp/admin/v1`: the operator API behind the admin panel | The design and roadmap are in [PLAN.md](../PLAN.md) at the repository root. diff --git a/docs/configuration.md b/docs/configuration.md index 59b0712..a299315 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -71,6 +71,10 @@ List-valued variables are comma-separated. A single `-` clears a list: | `media.pending_upload_expiry` | `WARP_MEDIA_PENDING_UPLOAD_EXPIRY` | `24h` | How long an ID reserved with `POST /media/v1/create` waits for its content. | | `media.max_pending_uploads` | `WARP_MEDIA_MAX_PENDING_UPLOADS` | `10` | How many of those one user may hold. | | `media.legacy_unauthenticated_downloads` | `WARP_MEDIA_LEGACY_UNAUTHENTICATED_DOWNLOADS` | `false` | Also serve the deprecated `/_matrix/media/v3/download` and `/thumbnail`, which need no token. Only for clients older than Matrix 1.11. | +| `push.enabled` | `WARP_PUSH_ENABLED` | `true` | Send pushes to the push gateways of users' pushers. Off, notifications are still counted. | +| `push.include_content` | `WARP_PUSH_INCLUDE_CONTENT` | `true` | Put the event's content into a push. Off, a push carries no message text. | +| `push.gateway_timeout` | `WARP_PUSH_GATEWAY_TIMEOUT` | `10s` | One request to a push gateway. | +| `push.notification_retention` | `WARP_PUSH_NOTIFICATION_RETENTION` | `720h` | How long a read notification is kept for `GET /notifications`. Unread ones are kept until read. | | `registration.enabled` | `WARP_REGISTRATION_ENABLED` | `false` | | | `registration.allow_runtime_toggle` | `WARP_REGISTRATION_ALLOW_RUNTIME_TOGGLE` | `true` | | | `registration.tokens` | `WARP_REGISTRATION_TOKENS` | `[]` | Non-empty = sign-up also needs one of these, as the `m.login.registration_token` stage. | diff --git a/docs/push.md b/docs/push.md new file mode 100644 index 0000000..83070ad --- /dev/null +++ b/docs/push.md @@ -0,0 +1,235 @@ +# Push and notifications + +`server/internal/pushrules` and `server/internal/pushserver`: PLAN.md +Phase 8. Push rules decide which events notify a user. Every event that +notifies somebody is recorded, counted per room as unread, and pushed to +the push gateways of that user's devices. + +The two packages split along a clear line: + +| Package | What | +|---|---| +| `pushrules` | The rule language: the rule model, the spec's default ruleset, conditions, glob matching, evaluation. Pure code, like `internal/matrix`: no storage, no HTTP. | +| `pushserver` | Everything around it: keeping each user's rules, the pushers, the worker that evaluates events, the counts, the push gateway client. | + +## Push rules + +### Where they are kept + +A user's push rules are their `m.push_rules` account data. That is where +the spec has clients read them, and where every client already takes them +from: the account data in its first `/sync`. A separate table would be a +second copy, re-rendered into account data on every change. + +The push server is the only writer. `PUT /user/{id}/account_data/m.push_rules` +is refused with `405 M_BAD_JSON`, as the spec requires. Changes go through +`/pushrules`. + +The stored document is the whole ruleset as a client sees it. When it is +read back, only the parts the user owns are kept: + +- the rules they created, in their order +- the `enabled` flag and `actions` of each default rule + +Everything else comes from `pushrules.Defaults`: which default rules exist, +and their conditions. So when a server upgrade adds a default rule (newer +spec versions do), every user gets it on their next read. Nobody's account +data has to be rewritten. A default rule the stored copy still has but +the server has since dropped is ignored. + +A user who has never changed a rule has nothing stored. They still have +rules: `/pushrules/`, the account data endpoint and the initial sync all +render the defaults for them. + +### Order + +Within a kind, user rules come before the defaults of that kind. The +exception is `.m.rule.master`, which stays ahead of everything, so switching +it on still silences everything. This is also where Synapse puts them. + +| Kind | Order | +|---|---| +| override | `.m.rule.master`, user rules, the other defaults | +| content | user rules, `.m.rule.contains_user_name` | +| room, sender | user rules | +| underride | user rules, the defaults | + +A new rule with no `before` or `after` goes first among the user rules of +its kind. A rule replaced with no `before` or `after` keeps its place and +its `enabled` flag. `before` and `after` must name a user rule of the same +kind; a default rule, or one that does not exist, is `404 M_NOT_FOUND`. + +### The defaults + +The spec's predefined rules, in the spec's order: + +- override: `master` (off), `suppress_notices`, `invite_for_me`, + `member_event`, `is_user_mention`, `contains_display_name`, + `is_room_mention`, `roomnotif`, `tombstone`, `reaction`, `server_acl`, + `suppress_edits` +- content: `contains_user_name` +- underride: `call`, `encrypted_room_one_to_one`, `room_one_to_one`, + `message`, `encrypted` + +An event with an `m.mentions` property says outright whom it mentions. +For such an event the three legacy rules that guess from the body — +`contains_display_name`, `contains_user_name` and `roomnotif` — are skipped, +as the spec requires. + +### Evaluation + +| Condition | Matches when | +|---|---| +| `event_match` | the string at `key` matches the glob, case-insensitively. `*` is any run of characters and `?` one. The whole value has to match, except for `content.body`, where the pattern has to match whole words somewhere in it. A number or boolean at the key never matches. | +| `event_property_is` | the value at `key` is exactly `value`: a string, an integer, a boolean or `null` | +| `event_property_contains` | the array at `key` contains `value` | +| `contains_display_name` | the body contains the user's display name in the room as whole words. The name is a literal, not a glob, so a user called `*` is not mentioned by everything. | +| `room_member_count` | the joined member count compares with `is` (`2`, `==2`, `<2`, `>2`, `<=2`, `>=2`) | +| `sender_notification_permission` | the sender's power level is at least `notifications.{key}` in the room's power levels (default 50) | +| anything else | never: a rule with a condition the server does not know never matches | + +Dotted keys follow the spec's escaping: `content.m\.mentions.user_ids` +reaches into a property whose name has a dot in it. + +A word boundary is any character that is not a letter, a digit or an +underscore, in any script. That is Python's `\w`, so it matches what +Synapse does. + +Compiled patterns are cached, up to 4096 of them. Patterns come from users, +so the cache is bounded and simply starts over when full. + +## Notifications + +### The worker + +The push server is a listener on the roomserver's emit stage, like the +federation queue, and like the queue it does no work there. The emit stage +runs while the room's write lock is held. Evaluating an event needs the +room's members, their display names, the power levels, and a ruleset for +each member, so the event goes on a list and a worker of the push server's +own takes it from there, in order. + +For each event the worker: + +1. If it is a local user leaving or being banned, marks everything in the + room read for them. A count left in a room they are no longer in would + stay on their badge for good. +2. Reads the room (`roomserver.PushRoomFor`): joined members and their + display names, the name and canonical alias, the power levels. +3. Evaluates the event for every local member except the sender, and for + the target of an invite, who is not a member yet. +4. Records every event that notifies somebody, in one transaction that also + moves each (user, room) count. +5. Wakes the syncs of the users it notified. +6. Pushes to their pushers' gateways, if `push.enabled` is on. + +The room is read as it is now, not as it was at the event. The worker runs +moments after the event lands, so the two only differ if somebody joined or +renamed themselves in between. Reading the state at each event would cost a +state resolution per message per member. + +Each user's merged ruleset is cached in memory. The push server makes every +write to the rules, which is what makes the cache safe in one process. + +### Counts + +| Table (migration `0010_push.sql`) | What | +|---|---| +| `push_notifications` | one row per event that notified a user: the event, its position in the room event stream, the actions, highlight, read | +| `push_room_counts` | each user's unread and highlight counts in each room, kept current by the writes, stamped with a notification stream position | +| `push_pushers` | pushers | +| `push_sequences` | the notification stream's counter | + +The counts are a stream of their own: the seventh position of a sync token. +Every write that changes a user's counts in a room stamps the room with a +new position in the same transaction. An incremental sync then asks which +rooms' counts moved since its token, and reports them even when nothing +else happened in them. + +That stream is what makes an asynchronous worker safe. A sync can show a +message before the worker has counted it. When the worker's write lands, it +moves the stream and wakes the sync, and the next sync carries the +corrected count. It also covers the case where counts move without any +event: a receipt sent from another device. + +`unread_notifications` is reported for every joined room in a response, not +only the rooms whose counts moved. A client showing a new message wants the +count next to it, whichever sync moved it. + +### Reading + +A read receipt — `m.read`, or `m.read.private`, which is private from other +users but not from the user's own badge — marks the user's notifications +in the room read, up to and including the event it names. The counts are +then recounted from the rows that are still unread rather than decremented, +so if anything ever put them out of step, the next receipt puts them right. +`/read_markers` goes through the same path. + +Notifications are not tracked per thread (threads are Phase 10). So a +receipt counts when it is for the whole room or for its `main` timeline. A +receipt inside a thread changes nothing; thread notifications are cleared +when the main receipt moves past them. + +Read notifications are kept for `push.notification_retention` (30 days by +default) for `GET /notifications`, then pruned hourly. Unread ones are kept +until they are read. + +## Pushers and the gateway + +`POST /pushers/set` accepts the `http` kind only. An `email` pusher would +need a mail server and templates, and a client asking for one is told +`400 M_INVALID_PARAM` rather than given a pusher that never fires. + +The gateway URL must be absolute, must have the path +`/_matrix/push/v1/notify`, and must not carry credentials. The spec asks for +HTTPS; plain HTTP is also accepted, because the usual deployment runs the +gateway (Sygnal, or a Warp-native one later — PLAN.md open question 6) next +to the homeserver on a private network. + +A pusher remembers the session that registered it. Logging that session out +or deleting the device removes the pusher: the userapi's device observer +now fans out to the keyserver and the push server. Registering the same app +ID and push key for another user removes it from the previous user, unless +`append` is set. + +A push carries the event's IDs, type, sender, sender display name, room +name and alias, the content (unless `push.include_content` is off), the +user's total unread count, the matched rule's tweaks, and the pusher's +`data` minus its `url`. `prio` is `high` for a highlight or anything that +makes a sound, and `low` otherwise. A pusher with `"format": "event_id_only"` +gets only the IDs, the priority and the counts; the app fetches the event +itself. + +Pushes go out from four senders fed by a queue of 1024, so a slow gateway +delays only its own pushes. A server error, a 429 or a network failure is +retried twice more, after 1 s and then 4 s. A 4xx is not retried. When the +queue is full, a push is dropped and logged. A push is a best-effort +wake-up: the notification is already counted, and the device sees it on +its next sync whether or not the push arrived. + +A push key the gateway lists in `rejected` is removed, so it is not pushed +to again. + +## Endpoints + +| Endpoint | Notes | +|---|---| +| `GET /pushrules/` · `GET /pushrules/global/` | The ruleset, under `global` or bare. The forms without the trailing slash are served too. | +| `GET` / `PUT` / `DELETE /pushrules/global/{kind}/{ruleId}` | `PUT` takes `?before=` / `?after=`. IDs starting with `.` are reserved (`400`). Deleting a default rule is `404`. | +| `GET` / `PUT /pushrules/global/{kind}/{ruleId}/enabled` | default rules included | +| `GET` / `PUT /pushrules/global/{kind}/{ruleId}/actions` | default rules included | +| `GET /pushers` · `POST /pushers/set` | `kind: null` removes a pusher | +| `GET /notifications` | `?from=`, `?limit=` (20 by default, at most 100), `?only=highlight`. Newest first. A notification whose event has since been purged, or that the user can no longer see, is left out. | +| `GET /user/{id}/account_data/m.push_rules` | the rendered ruleset | +| `PUT /user/{id}/account_data/m.push_rules` | `405 M_BAD_JSON` | + +## What is deliberately missing + +| Thing | Why | +|---|---| +| `unread_thread_notifications` | needs threads (Phase 10) | +| Skipping senders in `m.ignored_user_list` | ignored users' events still notify; a client hides them, but they count | +| Email pushers | needs a mail server; see above | +| Durable evaluation | the emit stage is in memory, so events persisted just before a crash are never evaluated. Their messages are still there; only the count and the push are missed. | +| A gateway URL deny-list | push gateways commonly sit on private networks, so private addresses cannot simply be refused the way remote media is. An operator who needs it should firewall outbound traffic. | +| Retries that survive a restart | a push waiting for its retry is lost on shutdown, for the same reason as above | diff --git a/docs/sync.md b/docs/sync.md index 6306297..26bfde5 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -42,7 +42,7 @@ second place for them to disagree. ## Stream tokens -A sync response is assembled from five streams that move independently: +A sync response is assembled from seven streams that move independently: | Stream | Moves when | Position from | |---|---|---| @@ -51,17 +51,21 @@ A sync response is assembled from five streams that move independently: | to-device | a device is sent a message | `sync_sequences` | | account data | a user changes a setting | `sync_sequences` | | presence | somebody's presence actually changes | `sync_sequences` | +| device list | somebody's end-to-end keys change ([e2ee.md](e2ee.md)) | `keyserver_sequences` | +| notification | a user's unread counts in a room move ([push.md](push.md)) | `push_sequences` | A client that has seen receipt 7 but only event 4 has to be able to say so in the one opaque string the API gives it, so a token names a position in each: ``` -s{pdu}_{receipt}_{to_device}_{account_data}_{presence} +s{pdu}_{receipt}_{to_device}_{account_data}_{presence}_{device_list}_{notification} ``` Trailing zeroes are left off, so a client that has only ever seen room -events carries `s12` rather than `s12_0_0_0_0`. A token with fewer +events carries `s12` rather than `s12_0_0_0_0_0_0`. The device list and +notification positions came later (Phases 7 and 8) and were added at the +end. A token with fewer positions than this build writes is accepted and the missing ones read as zero — which is what lets a token stored before a stream existed go on working: the client is sent that stream in full once and is in step @@ -400,7 +404,7 @@ no timestamp and have to decide what that meant. | Thing | Needs | Phase | |---|---|---| -| `unread_notifications`, `unread_thread_notifications` | push rules | 8 | +| `unread_thread_notifications` | threads | 10 | | Remote presence | presence over federation | — | | Threads, and threaded receipts beyond storing `thread_id` | threads | 10 | | `contains_url`, `event_fields` | nothing; they are parsed and ignored | — | @@ -410,9 +414,15 @@ no timestamp and have to decide what that meant. `device_lists` is sent as two empty lists rather than left out: a client reads a missing one as a malformed response, where two empty lists say "nothing changed". How the two lists are worked out, and the key counts -that ride along with them, are in [e2ee.md](e2ee.md). The notification -counts are left out entirely instead, because a zero there is a claim — -"you have nothing unread" — and an absent field is not. +that ride along with them, are in [e2ee.md](e2ee.md). + +Each joined room carries `unread_notifications` from the push server, +whose counts are the seventh stream, and the global account data carries +`m.push_rules`, rendered over the server defaults for a user who never +changed a rule. Both are described in [push.md](push.md). A build with no +push server leaves the counts out entirely rather than sending zeroes, +because a zero is a claim — "you have nothing unread" — and an absent field +is not. ## Endpoints diff --git a/server/internal/clientapi/accountdata.go b/server/internal/clientapi/accountdata.go index ad6f6a1..875e6a3 100644 --- a/server/internal/clientapi/accountdata.go +++ b/server/internal/clientapi/accountdata.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/dariuszjaniak/warp/server/internal/httputil" + "github.com/dariuszjaniak/warp/server/internal/pushserver" "github.com/dariuszjaniak/warp/server/internal/userapi" ) @@ -48,7 +49,14 @@ func (c *ClientAPI) getAccountData(w http.ResponseWriter, r *http.Request, auth return httputil.MissingParam("An account data type is required") } - content, err := c.users.AccountData(r.Context(), localpart, roomID, dataType) + var content []byte + if roomID == "" && dataType == pushserver.AccountDataType && c.push != nil { + // The push server's, rendered over the defaults: a user who never + // changed a rule has nothing stored and still has rules. + content, err = c.push.Document(r.Context(), localpart, auth.UserID) + } else { + content, err = c.users.AccountData(r.Context(), localpart, roomID, dataType) + } switch { case errors.Is(err, userapi.ErrNoSuchAccountData): return httputil.NotFound("Account data not found") @@ -93,6 +101,10 @@ func (c *ClientAPI) putAccountData(w http.ResponseWriter, r *http.Request, auth if dataType == typeFullyRead { return httputil.BadJSON("%s may only be set through /rooms/{roomId}/read_markers", typeFullyRead) } + if dataType == pushserver.AccountDataType { + return httputil.NewError(http.StatusMethodNotAllowed, httputil.ErrCodeBadJSON, + "%s may only be changed through the /pushrules endpoints", pushserver.AccountDataType) + } // The content is stored as the client sent it, so it is read as raw // JSON and only checked for being a JSON object at all. @@ -114,6 +126,11 @@ func (c *ClientAPI) putAccountData(w http.ResponseWriter, r *http.Request, auth if err := c.users.SetAccountData(r.Context(), localpart, roomID, dataType, content); err != nil { return err } + // Account data reaches a client's other devices through /sync, and + // one of them blocked in a long poll should hear about it now. + if c.sync != nil { + c.sync.Notifier().NotifyUser(auth.UserID) + } httputil.WriteJSON(w, http.StatusOK, struct{}{}) return nil } diff --git a/server/internal/clientapi/clientapi.go b/server/internal/clientapi/clientapi.go index b4d0ff2..0915645 100644 --- a/server/internal/clientapi/clientapi.go +++ b/server/internal/clientapi/clientapi.go @@ -22,6 +22,7 @@ import ( "github.com/dariuszjaniak/warp/server/internal/httputil" "github.com/dariuszjaniak/warp/server/internal/keyserver" "github.com/dariuszjaniak/warp/server/internal/mediaapi" + "github.com/dariuszjaniak/warp/server/internal/pushserver" "github.com/dariuszjaniak/warp/server/internal/ratelimit" "github.com/dariuszjaniak/warp/server/internal/roomserver" "github.com/dariuszjaniak/warp/server/internal/settings" @@ -38,6 +39,7 @@ type ClientAPI struct { sync *syncapi.API keys *keyserver.API media *mediaapi.API + push *pushserver.API legacyMedia bool settings *settings.Service federation *federation.Gate @@ -103,6 +105,10 @@ type Options struct { // answer M_UNRECOGNIZED. Media *mediaapi.API + // Push is the push server. Without it the push rule, pusher and + // notification endpoints answer M_UNRECOGNIZED. + Push *pushserver.API + // LegacyMediaDownloads serves the deprecated unauthenticated download // and thumbnail endpoints too. LegacyMediaDownloads bool @@ -162,6 +168,7 @@ func New(opts Options) (*ClientAPI, error) { sync: opts.Sync, keys: opts.Keys, media: opts.Media, + push: opts.Push, legacyMedia: opts.LegacyMediaDownloads, settings: opts.Settings, federation: opts.Federation, @@ -257,6 +264,7 @@ func (c *ClientAPI) RegisterRoutes(r *httputil.Router) { c.registerSyncRoutes(r) c.registerKeyRoutes(r) c.registerMediaRoutes(r) + c.registerPushRoutes(r) } // registerSyncRoutes adds the endpoints the syncapi serves, if there is diff --git a/server/internal/clientapi/push.go b/server/internal/clientapi/push.go new file mode 100644 index 0000000..926664a --- /dev/null +++ b/server/internal/clientapi/push.go @@ -0,0 +1,303 @@ +package clientapi + +import ( + "encoding/json" + "errors" + "net/http" + "strconv" + + "github.com/dariuszjaniak/warp/server/internal/httputil" + "github.com/dariuszjaniak/warp/server/internal/pushrules" + "github.com/dariuszjaniak/warp/server/internal/pushserver" + "github.com/dariuszjaniak/warp/server/internal/userapi" +) + +// The push endpoints: push rules, pushers and the notification list. +// Push rules are also the m.push_rules account data, which a client may +// read through the account data endpoints but not write through them. + +// pushRulesScope is the one scope Matrix has left. The spec once had a +// "device" scope too; it was never implemented anywhere and has been +// removed. +const pushRulesScope = "global" + +// registerPushRoutes adds the push endpoints, if there is a push server. +func (c *ClientAPI) registerPushRoutes(r *httputil.Router) { + if c.push == nil { + return + } + // The ruleset is at /pushrules/ with the slash; the form without it is + // served too, because clients send both. + c.authed(r, "GET "+v3+"/pushrules/{$}", c.limitDefault, c.handleGetPushRules) + c.authed(r, "GET "+v3+"/pushrules", c.limitDefault, c.handleGetPushRules) + c.authed(r, "GET "+v3+"/pushrules/{scope}/{$}", c.limitDefault, c.handleGetPushRuleScope) + c.authed(r, "GET "+v3+"/pushrules/{scope}", c.limitDefault, c.handleGetPushRuleScope) + c.authed(r, "GET "+v3+"/pushrules/{scope}/{kind}/{ruleID}", c.limitDefault, c.handleGetPushRule) + c.authed(r, "PUT "+v3+"/pushrules/{scope}/{kind}/{ruleID}", c.limitDefault, c.handlePutPushRule) + c.authed(r, "DELETE "+v3+"/pushrules/{scope}/{kind}/{ruleID}", c.limitDefault, c.handleDeletePushRule) + c.authed(r, "GET "+v3+"/pushrules/{scope}/{kind}/{ruleID}/{attr}", c.limitDefault, c.handleGetPushRuleAttr) + c.authed(r, "PUT "+v3+"/pushrules/{scope}/{kind}/{ruleID}/{attr}", c.limitDefault, c.handlePutPushRuleAttr) + + c.authed(r, "GET "+v3+"/pushers", c.limitDefault, c.handleGetPushers) + c.authed(r, "POST "+v3+"/pushers/set", c.limitDefault, c.handleSetPusher) + + c.authed(r, "GET "+v3+"/notifications", c.limitDefault, c.handleGetNotifications) +} + +// handleGetPushRules implements GET /_matrix/client/v3/pushrules/. +func (c *ClientAPI) handleGetPushRules(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + rs, err := c.push.Ruleset(r.Context(), auth.Localpart, auth.UserID) + if err != nil { + return err + } + httputil.WriteJSON(w, http.StatusOK, pushrules.Document{Global: rs}) + return nil +} + +// handleGetPushRuleScope implements GET /_matrix/client/v3/pushrules/global/. +func (c *ClientAPI) handleGetPushRuleScope(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + if err := checkPushScope(r); err != nil { + return err + } + rs, err := c.push.Ruleset(r.Context(), auth.Localpart, auth.UserID) + if err != nil { + return err + } + httputil.WriteJSON(w, http.StatusOK, rs) + return nil +} + +// handleGetPushRule implements GET /_matrix/client/v3/pushrules/global/{kind}/{ruleId}. +func (c *ClientAPI) handleGetPushRule(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + kind, ruleID, err := pushRulePath(r) + if err != nil { + return err + } + rule, err := c.push.Rule(r.Context(), auth.Localpart, auth.UserID, kind, ruleID) + if err != nil { + return pushRuleError(err) + } + httputil.WriteJSON(w, http.StatusOK, rule) + return nil +} + +// handlePutPushRule implements PUT /_matrix/client/v3/pushrules/global/{kind}/{ruleId}. +func (c *ClientAPI) handlePutPushRule(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + kind, ruleID, err := pushRulePath(r) + if err != nil { + return err + } + var body pushrules.NewRule + if err := httputil.DecodeJSON(w, r, &body); err != nil { + return err + } + query := r.URL.Query() + err = c.push.PutRule(r.Context(), auth.Localpart, auth.UserID, kind, ruleID, body, + query.Get("before"), query.Get("after")) + if err != nil { + return pushRuleError(err) + } + httputil.WriteJSON(w, http.StatusOK, struct{}{}) + return nil +} + +// handleDeletePushRule implements DELETE /_matrix/client/v3/pushrules/global/{kind}/{ruleId}. +func (c *ClientAPI) handleDeletePushRule(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + kind, ruleID, err := pushRulePath(r) + if err != nil { + return err + } + if err := c.push.DeleteRule(r.Context(), auth.Localpart, auth.UserID, kind, ruleID); err != nil { + return pushRuleError(err) + } + httputil.WriteJSON(w, http.StatusOK, struct{}{}) + return nil +} + +// The two attributes of a rule a client may read and write on their own. +const ( + pushRuleAttrEnabled = "enabled" + pushRuleAttrActions = "actions" +) + +// handleGetPushRuleAttr implements GET .../{ruleId}/enabled and .../{ruleId}/actions. +func (c *ClientAPI) handleGetPushRuleAttr(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + kind, ruleID, err := pushRulePath(r) + if err != nil { + return err + } + attr := r.PathValue("attr") + if attr != pushRuleAttrEnabled && attr != pushRuleAttrActions { + return httputil.Unrecognized() + } + rule, err := c.push.Rule(r.Context(), auth.Localpart, auth.UserID, kind, ruleID) + if err != nil { + return pushRuleError(err) + } + if attr == pushRuleAttrEnabled { + httputil.WriteJSON(w, http.StatusOK, map[string]bool{"enabled": rule.Enabled}) + return nil + } + httputil.WriteJSON(w, http.StatusOK, map[string][]json.RawMessage{"actions": rule.Actions}) + return nil +} + +// handlePutPushRuleAttr implements PUT .../{ruleId}/enabled and .../{ruleId}/actions. +func (c *ClientAPI) handlePutPushRuleAttr(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + kind, ruleID, err := pushRulePath(r) + if err != nil { + return err + } + switch r.PathValue("attr") { + case pushRuleAttrEnabled: + var body struct { + Enabled *bool `json:"enabled"` + } + if err := httputil.DecodeJSON(w, r, &body); err != nil { + return err + } + if body.Enabled == nil { + return httputil.MissingParam("enabled is required") + } + err = c.push.SetRuleEnabled(r.Context(), auth.Localpart, auth.UserID, kind, ruleID, *body.Enabled) + case pushRuleAttrActions: + var body struct { + Actions []json.RawMessage `json:"actions"` + } + if err := httputil.DecodeJSON(w, r, &body); err != nil { + return err + } + if body.Actions == nil { + return httputil.MissingParam("actions is required") + } + err = c.push.SetRuleActions(r.Context(), auth.Localpart, auth.UserID, kind, ruleID, body.Actions) + default: + return httputil.Unrecognized() + } + if err != nil { + return pushRuleError(err) + } + httputil.WriteJSON(w, http.StatusOK, struct{}{}) + return nil +} + +func checkPushScope(r *http.Request) error { + if scope := r.PathValue("scope"); scope != pushRulesScope { + return httputil.InvalidParam("%q is not a push rule scope; the only one is %q", scope, pushRulesScope) + } + return nil +} + +// pushRulePath reads the scope, kind and rule ID of a push rule path. +func pushRulePath(r *http.Request) (pushrules.Kind, string, error) { + if err := checkPushScope(r); err != nil { + return "", "", err + } + kind, ok := pushrules.ParseKind(r.PathValue("kind")) + if !ok { + return "", "", httputil.InvalidParam("%q is not a kind of push rule", r.PathValue("kind")) + } + ruleID := r.PathValue("ruleID") + if ruleID == "" { + return "", "", httputil.MissingParam("A rule ID is required") + } + return kind, ruleID, nil +} + +// pushRuleError maps the push server's errors onto the Matrix ones. +func pushRuleError(err error) error { + switch { + case errors.Is(err, pushserver.ErrNoSuchRule): + return httputil.NotFound("Push rule not found") + case errors.Is(err, pushserver.ErrNoSuchAnchor): + return httputil.NotFound("The rule named by before or after is not a rule of yours of this kind") + case errors.Is(err, pushserver.ErrInvalidRule): + return httputil.InvalidParam("%v", err) + } + return err +} + +// handleGetPushers implements GET /_matrix/client/v3/pushers. +func (c *ClientAPI) handleGetPushers(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + pushers, err := c.push.Pushers(r.Context(), auth.Localpart) + if err != nil { + return err + } + httputil.WriteJSON(w, http.StatusOK, map[string][]pushserver.Pusher{"pushers": pushers}) + return nil +} + +// handleSetPusher implements POST /_matrix/client/v3/pushers/set. +func (c *ClientAPI) handleSetPusher(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + // kind is read raw, because it has three meanings rather than two: + // absent is a malformed request, null removes the pusher, and a + // string sets one. + var body struct { + pushserver.SetPusherRequest + Kind json.RawMessage `json:"kind"` + } + if err := httputil.DecodeJSON(w, r, &body); err != nil { + return err + } + req := body.SetPusherRequest + switch string(body.Kind) { + case "": + return httputil.MissingParam("kind is required") + case "null": + req.Kind = nil + default: + var kind string + if err := json.Unmarshal(body.Kind, &kind); err != nil { + return httputil.BadJSON("kind must be a string or null") + } + req.Kind = &kind + } + + err := c.push.SetPusher(r.Context(), auth.Localpart, auth.DeviceID, req) + switch { + case errors.Is(err, pushserver.ErrMissingPusherField): + return httputil.MissingParam("%v", err) + case errors.Is(err, pushserver.ErrInvalidPusher): + return httputil.InvalidParam("%v", err) + case err != nil: + return err + } + httputil.WriteJSON(w, http.StatusOK, struct{}{}) + return nil +} + +// handleGetNotifications implements GET /_matrix/client/v3/notifications. +func (c *ClientAPI) handleGetNotifications(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + query := r.URL.Query() + req := pushserver.NotificationsRequest{ + UserID: auth.UserID, + From: query.Get("from"), + } + if raw := query.Get("limit"); raw != "" { + limit, err := strconv.Atoi(raw) + if err != nil || limit < 1 { + return httputil.InvalidParam("limit must be a positive integer") + } + req.Limit = limit + } + switch only := query.Get("only"); only { + case "": + case "highlight": + req.OnlyHighlight = true + default: + return httputil.InvalidParam("only must be highlight") + } + + notifications, next, err := c.push.Notifications(r.Context(), req) + if errors.Is(err, pushserver.ErrInvalidFrom) { + return httputil.InvalidParam("from is not a token this server handed out") + } + if err != nil { + return err + } + httputil.WriteJSON(w, http.StatusOK, struct { + Notifications []pushserver.Notification `json:"notifications"` + NextToken string `json:"next_token,omitempty"` + }{notifications, next}) + return nil +} diff --git a/server/internal/clientapi/sync_test.go b/server/internal/clientapi/sync_test.go index b51bae7..de690b6 100644 --- a/server/internal/clientapi/sync_test.go +++ b/server/internal/clientapi/sync_test.go @@ -260,7 +260,7 @@ func TestSyncRejectsAnUnreadableToken(t *testing.T) { // "s-1" is not here: a negative event position is a /messages token // from inside backfilled history, and a valid ?since=. Only the event // stream goes below zero. - for _, since := range []string{"nonsense", "s1_-1", "sx", "s1_2_3_4_5_6_7"} { + for _, since := range []string{"nonsense", "s1_-1", "sx", "s1_2_3_4_5_6_7_8"} { rec := ts.get("/_matrix/client/v3/sync?since="+url.QueryEscape(since), alice.AccessToken) expectError(t, rec, http.StatusBadRequest, "M_INVALID_PARAM") } diff --git a/server/internal/config/config.go b/server/internal/config/config.go index e473040..18310e8 100644 --- a/server/internal/config/config.go +++ b/server/internal/config/config.go @@ -25,6 +25,7 @@ type Config struct { Sync Sync `yaml:"sync"` Federation Federation `yaml:"federation"` Media Media `yaml:"media"` + Push Push `yaml:"push"` Registration Registration `yaml:"registration"` Logging Logging `yaml:"logging"` Metrics Metrics `yaml:"metrics"` @@ -285,6 +286,28 @@ type ThumbnailSize struct { Method string `yaml:"method"` } +// Push configures notifications and pushes to push gateways. +type Push struct { + // Enabled sends pushes to the push gateways of users' pushers. Off, + // notifications are still counted — a client's unread badges come + // from the counts, not from pushes — and nothing leaves the server. + Enabled bool `yaml:"enabled" env:"WARP_PUSH_ENABLED"` + + // IncludeContent puts the event's content into a push. Off, a push + // says who sent what kind of event in which room and nothing of what + // it said, for operators who would rather message text did not pass + // through a third-party push service. A pusher can ask for even less + // with the event_id_only format. + IncludeContent bool `yaml:"include_content" env:"WARP_PUSH_INCLUDE_CONTENT"` + + // GatewayTimeout bounds one request to a push gateway. + GatewayTimeout time.Duration `yaml:"gateway_timeout" env:"WARP_PUSH_GATEWAY_TIMEOUT"` + + // NotificationRetention is how long a notification that has been read + // is kept for GET /notifications. Unread ones are kept until read. + NotificationRetention time.Duration `yaml:"notification_retention" env:"WARP_PUSH_NOTIFICATION_RETENTION"` +} + // Registration controls open registration. type Registration struct { Enabled bool `yaml:"enabled" env:"WARP_REGISTRATION_ENABLED"` @@ -397,6 +420,12 @@ func Default() *Config { PendingUploadExpiry: 24 * time.Hour, MaxPendingUploads: 10, }, + Push: Push{ + Enabled: true, + IncludeContent: true, + GatewayTimeout: 10 * time.Second, + NotificationRetention: 30 * 24 * time.Hour, + }, Registration: Registration{ Enabled: false, AllowRuntimeToggle: true, @@ -552,6 +581,13 @@ func (c *Config) Validate() error { bad("media.max_pending_uploads: must be at least 1") } + if c.Push.GatewayTimeout <= 0 { + bad("push.gateway_timeout: must be positive") + } + if c.Push.NotificationRetention <= 0 { + bad("push.notification_retention: must be positive") + } + for i, token := range c.Registration.Tokens { if strings.TrimSpace(token) == "" { bad("registration.tokens[%d]: must not be empty", i) diff --git a/server/internal/pushrules/actions.go b/server/internal/pushrules/actions.go new file mode 100644 index 0000000..c14d399 --- /dev/null +++ b/server/internal/pushrules/actions.go @@ -0,0 +1,160 @@ +package pushrules + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "strings" +) + +// The action strings. dont_notify and coalesce are legacy: the spec has +// servers accept them and treat them as doing nothing, which for +// dont_notify is exactly what an empty action list means. +const ( + ActionNotify = "notify" + ActionDontNotify = "dont_notify" + ActionCoalesce = "coalesce" +) + +// The tweaks the server itself reads. Every other tweak goes to the push +// gateway as it was written. +const ( + TweakHighlight = "highlight" + TweakSound = "sound" +) + +// Outcome is what a matched rule's actions ask for. +type Outcome struct { + Notify bool + Highlight bool + + // Tweaks are every set_tweak of the actions, by name, as the push + // gateway is to be sent them. A highlight with no value is written + // out as true. + Tweaks map[string]json.RawMessage +} + +// ParseActions reads a rule's actions. +// +// A highlight tweak with no value means true, as the spec says. A +// highlight on a rule that does not notify is ignored: a count of +// highlighted notifications that included events that are not +// notifications at all would be no use to anybody. +func ParseActions(actions []json.RawMessage) Outcome { + out := Outcome{Tweaks: map[string]json.RawMessage{}} + for _, raw := range actions { + var s string + if json.Unmarshal(raw, &s) == nil { + if s == ActionNotify { + out.Notify = true + } + continue + } + var tweak struct { + SetTweak string `json:"set_tweak"` + Value json.RawMessage `json:"value"` + } + if json.Unmarshal(raw, &tweak) != nil || tweak.SetTweak == "" { + continue + } + value := tweak.Value + if tweak.SetTweak == TweakHighlight { + on := len(value) == 0 || bytes.Equal(bytes.TrimSpace(value), []byte("true")) + out.Highlight = on + value = json.RawMessage(fmt.Sprint(on)) + } + if len(value) == 0 { + continue + } + out.Tweaks[tweak.SetTweak] = value + } + if !out.Notify { + out.Highlight = false + } + return out +} + +// ErrInvalidRule is a rule or an action list a client may not store. +var ErrInvalidRule = errors.New("invalid push rule") + +func invalid(format string, args ...any) error { + return fmt.Errorf("%w: %s", ErrInvalidRule, fmt.Sprintf(format, args...)) +} + +// ValidateActions checks an action list a client sent. +func ValidateActions(actions []json.RawMessage) error { + if actions == nil { + return invalid("actions are required") + } + for _, raw := range actions { + var s string + if json.Unmarshal(raw, &s) == nil { + switch s { + case ActionNotify, ActionDontNotify, ActionCoalesce: + continue + } + return invalid("%q is not an action", s) + } + var tweak map[string]json.RawMessage + if json.Unmarshal(raw, &tweak) != nil { + return invalid("an action must be a string or an object") + } + var name string + if json.Unmarshal(tweak["set_tweak"], &name) != nil || name == "" { + return invalid("an object action must name a tweak in set_tweak") + } + } + return nil +} + +// NewRule is a rule a client asked to create or replace, before it has +// been given its kind's shape. +type NewRule struct { + Actions []json.RawMessage `json:"actions"` + Conditions *[]Condition `json:"conditions"` + Pattern *string `json:"pattern"` +} + +// Build checks a client's rule against its kind and returns the rule as +// stored. It is enabled: the spec has a new rule start that way. +func (n NewRule) Build(kind Kind, ruleID string) (Rule, error) { + switch { + case ruleID == "": + return Rule{}, invalid("a rule ID is required") + case IsReserved(ruleID): + return Rule{}, invalid("rule IDs starting with '.' are reserved for the server's own rules") + case strings.ContainsAny(ruleID, "/\\"): + return Rule{}, invalid("a rule ID may not contain a slash or a backslash") + } + if err := ValidateActions(n.Actions); err != nil { + return Rule{}, err + } + + rule := Rule{RuleID: ruleID, Enabled: true, Actions: n.Actions} + switch kind { + case KindOverride, KindUnderride: + if n.Conditions == nil { + return Rule{}, invalid("an %s rule needs conditions", kind) + } + for _, c := range *n.Conditions { + if c.Kind == "" { + return Rule{}, invalid("every condition needs a kind") + } + } + rule.Conditions = *n.Conditions + if rule.Conditions == nil { + rule.Conditions = []Condition{} + } + case KindContent: + if n.Pattern == nil { + return Rule{}, invalid("a content rule needs a pattern") + } + rule.Pattern = n.Pattern + case KindRoom, KindSender: + // The rule ID is the condition. + default: + return Rule{}, invalid("%q is not a kind of push rule", kind) + } + return rule, nil +} diff --git a/server/internal/pushrules/defaults.go b/server/internal/pushrules/defaults.go new file mode 100644 index 0000000..1c4a2ec --- /dev/null +++ b/server/internal/pushrules/defaults.go @@ -0,0 +1,132 @@ +package pushrules + +import ( + "encoding/json" + "strings" +) + +// The IDs of the server-default rules, from the spec's "Predefined Rules". +const ( + RuleMaster = ".m.rule.master" + RuleSuppressNotices = ".m.rule.suppress_notices" + RuleInviteForMe = ".m.rule.invite_for_me" + RuleMemberEvent = ".m.rule.member_event" + RuleIsUserMention = ".m.rule.is_user_mention" + RuleContainsDisplayName = ".m.rule.contains_display_name" + RuleIsRoomMention = ".m.rule.is_room_mention" + RuleRoomNotif = ".m.rule.roomnotif" + RuleTombstone = ".m.rule.tombstone" + RuleReaction = ".m.rule.reaction" + RuleServerACL = ".m.rule.server_acl" + RuleSuppressEdits = ".m.rule.suppress_edits" + + RuleContainsUserName = ".m.rule.contains_user_name" + + RuleCall = ".m.rule.call" + RuleEncryptedRoomOneToOne = ".m.rule.encrypted_room_one_to_one" + RuleRoomOneToOne = ".m.rule.room_one_to_one" + RuleMessage = ".m.rule.message" + RuleEncrypted = ".m.rule.encrypted" +) + +// legacyMentionRules are the default rules that guess at a mention from +// the body. An event carrying m.mentions says who it mentions outright, +// and the spec has these skipped for it: otherwise a reply quoting +// somebody's name would ping them although its sender chose not to. +var legacyMentionRules = map[string]bool{ + RuleContainsDisplayName: true, + RuleContainsUserName: true, + RuleRoomNotif: true, +} + +// The actions the defaults use, spelled once. +var ( + actNotify = json.RawMessage(`"notify"`) + actSoundDefault = json.RawMessage(`{"set_tweak":"sound","value":"default"}`) + actSoundRing = json.RawMessage(`{"set_tweak":"sound","value":"ring"}`) + actHighlight = json.RawMessage(`{"set_tweak":"highlight"}`) +) + +func match(key, pattern string) Condition { + return Condition{Kind: CondEventMatch, Key: key, Pattern: &pattern} +} + +func propertyIs(key string, value json.RawMessage) Condition { + return Condition{Kind: CondEventPropertyIs, Key: key, Value: value} +} + +func actions(a ...json.RawMessage) []json.RawMessage { return a } + +func override(id string, enabled bool, acts []json.RawMessage, conds ...Condition) Rule { + if conds == nil { + conds = []Condition{} + } + return Rule{RuleID: id, Default: true, Enabled: enabled, Actions: acts, Conditions: conds} +} + +// Defaults returns the server-default ruleset for a user, in the spec's +// order. It is built per user because three of the rules name the user: +// the invite rule matches their user ID, the mention rule looks for it in +// m.mentions, and the content rule looks for their localpart in the body. +func Defaults(userID string) Ruleset { + localpart := strings.TrimPrefix(userID, "@") + if i := strings.IndexByte(localpart, ':'); i >= 0 { + localpart = localpart[:i] + } + quotedUser, _ := json.Marshal(userID) + + rs := Ruleset{ + Override: []Rule{ + override(RuleMaster, false, actions()), + override(RuleSuppressNotices, true, actions(), + match("content.msgtype", "m.notice")), + override(RuleInviteForMe, true, actions(actNotify, actSoundDefault), + match("type", "m.room.member"), + match("content.membership", "invite"), + match("state_key", userID)), + override(RuleMemberEvent, true, actions(), + match("type", "m.room.member")), + override(RuleIsUserMention, true, actions(actNotify, actSoundDefault, actHighlight), + Condition{Kind: CondEventPropertyContains, Key: `content.m\.mentions.user_ids`, Value: quotedUser}), + override(RuleContainsDisplayName, true, actions(actNotify, actSoundDefault, actHighlight), + Condition{Kind: CondContainsDisplayName}), + override(RuleIsRoomMention, true, actions(actNotify, actHighlight), + propertyIs(`content.m\.mentions.room`, json.RawMessage(`true`)), + Condition{Kind: CondSenderNotificationPermission, Key: "room"}), + override(RuleRoomNotif, true, actions(actNotify, actHighlight), + Condition{Kind: CondSenderNotificationPermission, Key: "room"}, + match("content.body", "@room")), + override(RuleTombstone, true, actions(actNotify, actHighlight), + match("type", "m.room.tombstone"), + match("state_key", "")), + override(RuleReaction, true, actions(), + match("type", "m.reaction")), + override(RuleServerACL, true, actions(), + match("type", "m.room.server_acl"), + match("state_key", "")), + override(RuleSuppressEdits, true, actions(), + propertyIs(`content.m\.relates_to.rel_type`, json.RawMessage(`"m.replace"`))), + }, + Content: []Rule{{ + RuleID: RuleContainsUserName, Default: true, Enabled: true, + Actions: actions(actNotify, actSoundDefault, actHighlight), + Pattern: &localpart, + }}, + Underride: []Rule{ + override(RuleCall, true, actions(actNotify, actSoundRing), + match("type", "m.call.invite")), + override(RuleEncryptedRoomOneToOne, true, actions(actNotify, actSoundDefault), + Condition{Kind: CondRoomMemberCount, Is: "2"}, + match("type", "m.room.encrypted")), + override(RuleRoomOneToOne, true, actions(actNotify, actSoundDefault), + Condition{Kind: CondRoomMemberCount, Is: "2"}, + match("type", "m.room.message")), + override(RuleMessage, true, actions(actNotify), + match("type", "m.room.message")), + override(RuleEncrypted, true, actions(actNotify), + match("type", "m.room.encrypted")), + }, + } + rs.normalise() + return rs +} diff --git a/server/internal/pushrules/edit.go b/server/internal/pushrules/edit.go new file mode 100644 index 0000000..c6851ee --- /dev/null +++ b/server/internal/pushrules/edit.go @@ -0,0 +1,107 @@ +package pushrules + +import ( + "encoding/json" + "errors" + "slices" +) + +// Errors an edit returns for cases a caller has to tell apart. +var ( + // ErrNoSuchRule is a rule that does not exist. + ErrNoSuchRule = errors.New("no such push rule") + + // ErrNoSuchAnchor is a before or after naming a rule that is not a + // user rule of the same kind. + ErrNoSuchAnchor = errors.New("the rule to place this one next to does not exist") +) + +// Put creates or replaces a user rule. +// +// With before or after it goes next to that rule, which must be a user +// rule of the same kind. With neither, a new rule goes first among the +// user rules of its kind and a replaced one keeps its place and its +// enabled flag. Either way a user rule stays below .m.rule.master and above +// every other default of its kind, which is where Merge puts them. +func (rs *Ruleset) Put(kind Kind, rule Rule, before, after string) error { + if IsReserved(rule.RuleID) { + return invalid("rule IDs starting with '.' are reserved for the server's own rules") + } + if IsReserved(before) || IsReserved(after) { + return ErrNoSuchAnchor + } + rules := slices.Clone(rs.Rules(kind)) + + existing := slices.IndexFunc(rules, func(r Rule) bool { return r.RuleID == rule.RuleID && !r.Default }) + if existing >= 0 && before == "" && after == "" { + rule.Enabled = rules[existing].Enabled + rules[existing] = rule + rs.SetRules(kind, rules) + return nil + } + if existing >= 0 { + rules = slices.Delete(rules, existing, existing+1) + } + + var at int + switch { + case before != "" || after != "": + anchor := before + if anchor == "" { + anchor = after + } + i := slices.IndexFunc(rules, func(r Rule) bool { return r.RuleID == anchor && !r.Default }) + if i < 0 { + return ErrNoSuchAnchor + } + at = i + if before == "" { + at = i + 1 + } + default: + // First among the user rules: after any default that Merge puts + // ahead of them. + at = slices.IndexFunc(rules, func(r Rule) bool { return !r.Default || r.RuleID != RuleMaster }) + if at < 0 { + at = len(rules) + } + } + rs.SetRules(kind, slices.Insert(rules, at, rule)) + return nil +} + +// Delete removes a user rule. A default rule cannot be deleted, only +// switched off, and is reported as not there. +func (rs *Ruleset) Delete(kind Kind, ruleID string) error { + rules := rs.Rules(kind) + i := slices.IndexFunc(rules, func(r Rule) bool { return r.RuleID == ruleID && !r.Default }) + if i < 0 { + return ErrNoSuchRule + } + rs.SetRules(kind, slices.Delete(slices.Clone(rules), i, i+1)) + return nil +} + +// SetEnabled switches a rule on or off. Default rules included. +func (rs *Ruleset) SetEnabled(kind Kind, ruleID string, enabled bool) error { + return rs.update(kind, ruleID, func(r *Rule) { r.Enabled = enabled }) +} + +// SetActions replaces a rule's actions. Default rules included. +func (rs *Ruleset) SetActions(kind Kind, ruleID string, actions []json.RawMessage) error { + if err := ValidateActions(actions); err != nil { + return err + } + return rs.update(kind, ruleID, func(r *Rule) { r.Actions = actions }) +} + +func (rs *Ruleset) update(kind Kind, ruleID string, fn func(*Rule)) error { + rules := slices.Clone(rs.Rules(kind)) + i := slices.IndexFunc(rules, func(r Rule) bool { return r.RuleID == ruleID }) + if i < 0 { + return ErrNoSuchRule + } + fn(&rules[i]) + rs.SetRules(kind, rules) + return nil +} diff --git a/server/internal/pushrules/evaluate.go b/server/internal/pushrules/evaluate.go new file mode 100644 index 0000000..dd3ccf0 --- /dev/null +++ b/server/internal/pushrules/evaluate.go @@ -0,0 +1,360 @@ +package pushrules + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + "sync" +) + +// Event is an event as the push rules see it: a tree of JSON values that +// dotted keys are looked up in. +type Event struct { + tree map[string]any + + // hasMentions is whether content carries m.mentions at all, which + // switches the legacy mention rules off. + hasMentions bool +} + +// NewEvent parses an event. Any shape of event works — the federation +// form or the client one — because the keys rules look at (type, sender, +// state_key, room_id, content) are the same in both. +func NewEvent(raw []byte) (*Event, error) { + dec := json.NewDecoder(bytes.NewReader(raw)) + // Numbers stay as written, so that an integer compares as one: a float + // round trip would make 9007199254740993 equal to its neighbour. + dec.UseNumber() + var tree map[string]any + if err := dec.Decode(&tree); err != nil { + return nil, fmt.Errorf("pushrules: event is not a JSON object: %w", err) + } + ev := &Event{tree: tree} + if content, ok := tree["content"].(map[string]any); ok { + _, ev.hasMentions = content["m.mentions"] + } + return ev, nil +} + +// lookup follows a dotted key into the event. +func (e *Event) lookup(key string) (any, bool) { + var cur any = e.tree + for _, part := range SplitKey(key) { + obj, ok := cur.(map[string]any) + if !ok { + return nil, false + } + if cur, ok = obj[part]; !ok { + return nil, false + } + } + return cur, true +} + +// str is the string at a key, if there is one. +func (e *Event) str(key string) (string, bool) { + v, ok := e.lookup(key) + if !ok { + return "", false + } + s, ok := v.(string) + return s, ok +} + +// SplitKey splits a dotted key into its parts. A dot preceded by a +// backslash is part of a name rather than a separator, which is how a key +// reaches into `m.mentions`; a backslash preceded by one is a literal +// backslash. Any other backslash is kept as it is. +func SplitKey(key string) []string { + var parts []string + var cur strings.Builder + for i := 0; i < len(key); i++ { + c := key[i] + switch { + case c == '\\' && i+1 < len(key) && (key[i+1] == '.' || key[i+1] == '\\'): + cur.WriteByte(key[i+1]) + i++ + case c == '.': + parts = append(parts, cur.String()) + cur.Reset() + default: + cur.WriteByte(c) + } + } + return append(parts, cur.String()) +} + +// Context is what an evaluation needs to know beyond the event itself. +type Context struct { + // UserID is the user the rules belong to. + UserID string + + // DisplayName is the user's display name in the room, or empty. + DisplayName string + + // MemberCount is how many users are joined to the room. + MemberCount int + + // SenderMayNotify reports whether the event's sender has the power a + // notification key — "room", for @room — requires. Nil means nobody + // has. + SenderMayNotify func(key string) bool +} + +// Evaluate runs the ruleset against an event and returns the first +// enabled rule that matches it, kind by kind in priority order. +func (rs *Ruleset) Evaluate(ev *Event, ctx Context) (Rule, Kind, bool) { + for _, kind := range Kinds { + for _, rule := range rs.Rules(kind) { + if !rule.Enabled { + continue + } + if rule.Default && ev.hasMentions && legacyMentionRules[rule.RuleID] { + continue + } + if ruleMatches(kind, rule, ev, ctx) { + return rule, kind, true + } + } + } + return Rule{}, "", false +} + +func ruleMatches(kind Kind, rule Rule, ev *Event, ctx Context) bool { + switch kind { + case KindOverride, KindUnderride: + for _, cond := range rule.Conditions { + if !conditionMatches(cond, ev, ctx) { + return false + } + } + return true + case KindContent: + if rule.Pattern == nil { + return false + } + body, ok := ev.str("content.body") + return ok && globMatches(*rule.Pattern, body, true) + case KindRoom: + roomID, ok := ev.str("room_id") + return ok && roomID == rule.RuleID + case KindSender: + sender, ok := ev.str("sender") + return ok && sender == rule.RuleID + } + return false +} + +func conditionMatches(cond Condition, ev *Event, ctx Context) bool { + switch cond.Kind { + case CondEventMatch: + if cond.Pattern == nil { + return false + } + // event_match is about strings: a number or a boolean at the key + // is not a match for any pattern, "*" included. + value, ok := ev.str(cond.Key) + return ok && globMatches(*cond.Pattern, value, cond.Key == "content.body") + + case CondEventPropertyIs: + want, ok := scalar(cond.Value) + if !ok { + return false + } + got, ok := ev.lookup(cond.Key) + return ok && scalarEqual(got, want) + + case CondEventPropertyContains: + want, ok := scalar(cond.Value) + if !ok { + return false + } + got, ok := ev.lookup(cond.Key) + if !ok { + return false + } + list, ok := got.([]any) + if !ok { + return false + } + for _, item := range list { + if scalarEqual(item, want) { + return true + } + } + return false + + case CondContainsDisplayName: + if ctx.DisplayName == "" { + return false + } + body, ok := ev.str("content.body") + return ok && literalMatches(ctx.DisplayName, body) + + case CondRoomMemberCount: + return memberCountMatches(cond.Is, ctx.MemberCount) + + case CondSenderNotificationPermission: + return cond.Key != "" && ctx.SenderMayNotify != nil && ctx.SenderMayNotify(cond.Key) + } + return false +} + +// scalar decodes a condition's value. Only the canonical JSON scalars may +// be compared: a string, an integer, a boolean or null. +func scalar(raw json.RawMessage) (any, bool) { + if len(raw) == 0 { + return nil, false + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, false + } + switch v.(type) { + case string, bool, nil, json.Number: + return v, true + } + return nil, false +} + +// scalarEqual compares an event value with a condition value. Numbers are +// compared as integers, which is all canonical JSON allows. +func scalarEqual(got, want any) bool { + switch w := want.(type) { + case nil: + return got == nil + case string: + g, ok := got.(string) + return ok && g == w + case bool: + g, ok := got.(bool) + return ok && g == w + case json.Number: + g, ok := got.(json.Number) + if !ok { + return false + } + gi, err1 := g.Int64() + wi, err2 := w.Int64() + return err1 == nil && err2 == nil && gi == wi + } + return false +} + +// memberCountMatches evaluates room_member_count's "is": an optional +// comparison operator, then an integer. No operator means equality. +func memberCountMatches(is string, count int) bool { + op := "==" + for _, candidate := range []string{"==", "<=", ">=", "<", ">"} { + if rest, ok := strings.CutPrefix(is, candidate); ok { + op, is = candidate, rest + break + } + } + n, err := strconv.Atoi(is) + if err != nil { + return false + } + switch op { + case "==": + return count == n + case "<": + return count < n + case ">": + return count > n + case "<=": + return count <= n + case ">=": + return count >= n + } + return false +} + +// Glob matching. +// +// A pattern matches case-insensitively, with `*` for any run of characters +// and `?` for one. It normally has to match the whole value. Against +// content.body it instead has to match a run of whole words somewhere in +// it: a body is prose, and the rule that looks for "alice" should fire on +// "hi alice!" and not on "malice". + +// wordChar is the class a word boundary is the edge of: letters and digits +// in any script, and the underscore, as Python's \w — and so Synapse — +// has it. +const wordChar = `\p{L}\p{N}_` + +var ( + globCacheMu sync.Mutex + globCache = map[string]*regexp.Regexp{} +) + +// maxGlobCache bounds the compiled pattern cache. Patterns come from +// users, so an unbounded cache would be a way to fill the server's memory; +// past the bound it starts again, which costs a compile per pattern. +const maxGlobCache = 4096 + +func globMatches(pattern, value string, words bool) bool { + re := compileGlob(pattern, words) + return re != nil && re.MatchString(value) +} + +func compileGlob(pattern string, words bool) *regexp.Regexp { + key := "e\x00" + pattern + if words { + key = "w\x00" + pattern + } + + globCacheMu.Lock() + re, ok := globCache[key] + globCacheMu.Unlock() + if ok { + return re + } + + var b strings.Builder + for _, r := range pattern { + switch r { + case '*': + b.WriteString(".*?") + case '?': + b.WriteString(".") + default: + b.WriteString(regexp.QuoteMeta(string(r))) + } + } + re = boundaryRegexp(b.String(), words) + + globCacheMu.Lock() + if len(globCache) >= maxGlobCache { + globCache = map[string]*regexp.Regexp{} + } + globCache[key] = re + globCacheMu.Unlock() + return re +} + +// literalMatches is a word match of a literal string, which is what +// contains_display_name asks: a display name is not a pattern, and one +// called "*" must not match every message. +func literalMatches(literal, value string) bool { + re := boundaryRegexp(regexp.QuoteMeta(literal), true) + return re != nil && re.MatchString(value) +} + +func boundaryRegexp(expr string, words bool) *regexp.Regexp { + if words { + expr = `(?:^|[^` + wordChar + `])(?:` + expr + `)(?:[^` + wordChar + `]|$)` + } else { + expr = `^(?:` + expr + `)$` + } + re, err := regexp.Compile(`(?is)` + expr) + if err != nil { + return nil + } + return re +} diff --git a/server/internal/pushrules/pushrules_test.go b/server/internal/pushrules/pushrules_test.go new file mode 100644 index 0000000..bba71aa --- /dev/null +++ b/server/internal/pushrules/pushrules_test.go @@ -0,0 +1,388 @@ +package pushrules_test + +import ( + "encoding/json" + "errors" + "slices" + "testing" + + "github.com/dariuszjaniak/warp/server/internal/pushrules" +) + +const alice = "@alice:warp.example" + +func mustEvent(t *testing.T, raw string) *pushrules.Event { + t.Helper() + ev, err := pushrules.NewEvent([]byte(raw)) + if err != nil { + t.Fatal(err) + } + return ev +} + +func ruleIDs(rules []pushrules.Rule) []string { + var out []string + for _, r := range rules { + out = append(out, r.RuleID) + } + return out +} + +func TestSplitKey(t *testing.T) { + for key, want := range map[string][]string{ + "content.body": {"content", "body"}, + `content.m\.mentions.user_ids`: {"content", "m.mentions", "user_ids"}, + `a\\.b`: {`a\`, "b"}, + `a\b`: {`a\b`}, + "type": {"type"}, + } { + if got := pushrules.SplitKey(key); !slices.Equal(got, want) { + t.Errorf("SplitKey(%q) = %q, want %q", key, got, want) + } + } +} + +func TestDefaultsAreTheSpecsOrder(t *testing.T) { + rs := pushrules.Defaults(alice) + wantOverride := []string{ + ".m.rule.master", ".m.rule.suppress_notices", ".m.rule.invite_for_me", ".m.rule.member_event", + ".m.rule.is_user_mention", ".m.rule.contains_display_name", ".m.rule.is_room_mention", + ".m.rule.roomnotif", ".m.rule.tombstone", ".m.rule.reaction", ".m.rule.server_acl", + ".m.rule.suppress_edits", + } + if got := ruleIDs(rs.Override); !slices.Equal(got, wantOverride) { + t.Errorf("override = %v", got) + } + wantUnderride := []string{".m.rule.call", ".m.rule.encrypted_room_one_to_one", + ".m.rule.room_one_to_one", ".m.rule.message", ".m.rule.encrypted"} + if got := ruleIDs(rs.Underride); !slices.Equal(got, wantUnderride) { + t.Errorf("underride = %v", got) + } + if len(rs.Content) != 1 || *rs.Content[0].Pattern != "alice" { + t.Errorf("content = %+v", rs.Content) + } + + // Every kind is an array on the wire, empty ones included, and the + // master rule has an empty condition list rather than none. + raw, _ := json.Marshal(rs) + var wire map[string]json.RawMessage + _ = json.Unmarshal(raw, &wire) + if string(wire["room"]) != "[]" || string(wire["sender"]) != "[]" { + t.Errorf("empty kinds = %s, %s", wire["room"], wire["sender"]) + } + var override []map[string]json.RawMessage + _ = json.Unmarshal(wire["override"], &override) + if string(override[0]["conditions"]) != "[]" || string(override[0]["actions"]) != "[]" { + t.Errorf("master = %s", raw) + } +} + +func evaluate(t *testing.T, rs pushrules.Ruleset, raw string, ctx pushrules.Context) (string, pushrules.Outcome) { + t.Helper() + if ctx.UserID == "" { + ctx.UserID = alice + } + rule, _, ok := rs.Evaluate(mustEvent(t, raw), ctx) + if !ok { + return "", pushrules.Outcome{} + } + return rule.RuleID, pushrules.ParseActions(rule.Actions) +} + +func TestDefaultEvaluation(t *testing.T) { + rs := pushrules.Defaults(alice) + mayNotify := func(string) bool { return true } + group := pushrules.Context{MemberCount: 5, DisplayName: "Alice Liddell", SenderMayNotify: mayNotify} + dm := pushrules.Context{MemberCount: 2} + + cases := []struct { + name string + event string + ctx pushrules.Context + rule string + notify bool + highlight bool + }{ + {"a message in a group", `{"type":"m.room.message","sender":"@bob:x","content":{"body":"hello"}}`, group, + ".m.rule.message", true, false}, + {"a message in a DM", `{"type":"m.room.message","sender":"@bob:x","content":{"body":"hello"}}`, dm, + ".m.rule.room_one_to_one", true, false}, + {"an encrypted DM", `{"type":"m.room.encrypted","sender":"@bob:x","content":{}}`, dm, + ".m.rule.encrypted_room_one_to_one", true, false}, + {"a notice", `{"type":"m.room.message","sender":"@bob:x","content":{"msgtype":"m.notice","body":"alice"}}`, group, + ".m.rule.suppress_notices", false, false}, + {"the user's name at a word boundary", `{"type":"m.room.message","sender":"@bob:x","content":{"body":"hi ALICE!"}}`, group, + ".m.rule.contains_user_name", true, true}, + {"the user's name inside a word", `{"type":"m.room.message","sender":"@bob:x","content":{"body":"malice"}}`, group, + ".m.rule.message", true, false}, + {"the display name", `{"type":"m.room.message","sender":"@bob:x","content":{"body":"ask alice liddell."}}`, group, + ".m.rule.contains_display_name", true, true}, + {"an intentional mention", `{"type":"m.room.message","sender":"@bob:x","content":{"body":"x","m.mentions":{"user_ids":["@alice:warp.example"]}}}`, group, + ".m.rule.is_user_mention", true, true}, + {"m.mentions turns the body rules off", `{"type":"m.room.message","sender":"@bob:x","content":{"body":"alice","m.mentions":{}}}`, group, + ".m.rule.message", true, false}, + {"@room by somebody with the power", `{"type":"m.room.message","sender":"@bob:x","content":{"body":"@room look"}}`, group, + ".m.rule.roomnotif", true, true}, + {"@room by somebody without it", `{"type":"m.room.message","sender":"@bob:x","content":{"body":"@room look"}}`, pushrules.Context{MemberCount: 5}, + ".m.rule.message", true, false}, + {"a room mention", `{"type":"m.room.message","sender":"@bob:x","content":{"body":"x","m.mentions":{"room":true}}}`, group, + ".m.rule.is_room_mention", true, true}, + {"an invite for the user", `{"type":"m.room.member","state_key":"@alice:warp.example","sender":"@bob:x","content":{"membership":"invite"}}`, group, + ".m.rule.invite_for_me", true, false}, + {"somebody else's invite", `{"type":"m.room.member","state_key":"@carol:x","sender":"@bob:x","content":{"membership":"invite"}}`, group, + ".m.rule.member_event", false, false}, + {"an edit", `{"type":"m.room.message","sender":"@bob:x","content":{"body":"alice","m.relates_to":{"rel_type":"m.replace"}}}`, group, + ".m.rule.suppress_edits", false, false}, + {"a tombstone", `{"type":"m.room.tombstone","state_key":"","sender":"@bob:x","content":{}}`, group, + ".m.rule.tombstone", true, true}, + {"a call", `{"type":"m.call.invite","sender":"@bob:x","content":{}}`, group, + ".m.rule.call", true, false}, + {"a reaction", `{"type":"m.reaction","sender":"@bob:x","content":{}}`, group, + ".m.rule.reaction", false, false}, + {"something no rule is about", `{"type":"m.room.topic","state_key":"","sender":"@bob:x","content":{}}`, group, + "", false, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rule, outcome := evaluate(t, rs, tc.event, tc.ctx) + if rule != tc.rule || outcome.Notify != tc.notify || outcome.Highlight != tc.highlight { + t.Errorf("got %q notify=%v highlight=%v, want %q notify=%v highlight=%v", + rule, outcome.Notify, outcome.Highlight, tc.rule, tc.notify, tc.highlight) + } + }) + } +} + +func TestConditions(t *testing.T) { + pattern := func(s string) *string { return &s } + rule := func(conds ...pushrules.Condition) pushrules.Ruleset { + return pushrules.Ruleset{Override: []pushrules.Rule{{ + RuleID: "r", Enabled: true, Actions: []json.RawMessage{json.RawMessage(`"notify"`)}, Conditions: conds, + }}} + } + ev := `{"type":"org.example","sender":"@bob:x","room_id":"!r:x","content":{"n":3,"b":true,"z":null,"s":"Hello World","list":["a",1,true],"m.dotted":"yes"}}` + + for _, tc := range []struct { + name string + cond pushrules.Condition + want bool + }{ + {"glob star", pushrules.Condition{Kind: "event_match", Key: "type", Pattern: pattern("org.*")}, true}, + {"glob question mark", pushrules.Condition{Kind: "event_match", Key: "type", Pattern: pattern("org.exampl?")}, true}, + {"globs match the whole value", pushrules.Condition{Kind: "event_match", Key: "type", Pattern: pattern("org")}, false}, + {"globs are case-insensitive", pushrules.Condition{Kind: "event_match", Key: "content.s", Pattern: pattern("hello world")}, true}, + {"globs only match strings", pushrules.Condition{Kind: "event_match", Key: "content.n", Pattern: pattern("*")}, false}, + {"a missing key", pushrules.Condition{Kind: "event_match", Key: "content.nope", Pattern: pattern("*")}, false}, + {"an escaped dot", pushrules.Condition{Kind: "event_match", Key: `content.m\.dotted`, Pattern: pattern("yes")}, true}, + {"property is an integer", pushrules.Condition{Kind: "event_property_is", Key: "content.n", Value: json.RawMessage(`3`)}, true}, + {"property is another integer", pushrules.Condition{Kind: "event_property_is", Key: "content.n", Value: json.RawMessage(`4`)}, false}, + {"property is a string, exactly", pushrules.Condition{Kind: "event_property_is", Key: "content.s", Value: json.RawMessage(`"hello world"`)}, false}, + {"property is true", pushrules.Condition{Kind: "event_property_is", Key: "content.b", Value: json.RawMessage(`true`)}, true}, + {"property is null", pushrules.Condition{Kind: "event_property_is", Key: "content.z", Value: json.RawMessage(`null`)}, true}, + {"null is not a missing key", pushrules.Condition{Kind: "event_property_is", Key: "content.nope", Value: json.RawMessage(`null`)}, false}, + {"property contains", pushrules.Condition{Kind: "event_property_contains", Key: "content.list", Value: json.RawMessage(`1`)}, true}, + {"property does not contain", pushrules.Condition{Kind: "event_property_contains", Key: "content.list", Value: json.RawMessage(`"b"`)}, false}, + {"contains on a non-list", pushrules.Condition{Kind: "event_property_contains", Key: "content.s", Value: json.RawMessage(`"H"`)}, false}, + {"member count equal", pushrules.Condition{Kind: "room_member_count", Is: "5"}, true}, + {"member count ==", pushrules.Condition{Kind: "room_member_count", Is: "==5"}, true}, + {"member count <", pushrules.Condition{Kind: "room_member_count", Is: "<5"}, false}, + {"member count >=", pushrules.Condition{Kind: "room_member_count", Is: ">=5"}, true}, + {"member count garbage", pushrules.Condition{Kind: "room_member_count", Is: "lots"}, false}, + {"an unknown condition", pushrules.Condition{Kind: "org.example.condition"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + got, _ := evaluate(t, rule(tc.cond), ev, pushrules.Context{MemberCount: 5}) + if (got == "r") != tc.want { + t.Errorf("matched = %v, want %v", got == "r", tc.want) + } + }) + } + + // A display name is a literal, not a glob. + got, _ := evaluate(t, rule(pushrules.Condition{Kind: "contains_display_name"}), + `{"type":"m.room.message","content":{"body":"anything"}}`, pushrules.Context{DisplayName: "*"}) + if got != "" { + t.Error("a display name of * matched an unrelated body") + } +} + +func TestRoomAndSenderRules(t *testing.T) { + rs := pushrules.Defaults(alice) + quiet := pushrules.Rule{RuleID: "!quiet:x", Enabled: true, Actions: []json.RawMessage{}} + if err := rs.Put(pushrules.KindRoom, quiet, "", ""); err != nil { + t.Fatal(err) + } + loud := pushrules.Rule{RuleID: "@boss:x", Enabled: true, + Actions: []json.RawMessage{json.RawMessage(`"notify"`), json.RawMessage(`{"set_tweak":"highlight"}`)}} + if err := rs.Put(pushrules.KindSender, loud, "", ""); err != nil { + t.Fatal(err) + } + + if rule, out := evaluate(t, rs, `{"type":"m.room.message","room_id":"!quiet:x","sender":"@boss:x","content":{"body":"x"}}`, + pushrules.Context{MemberCount: 3}); rule != "!quiet:x" || out.Notify { + t.Errorf("room rule: %q %+v", rule, out) + } + if rule, out := evaluate(t, rs, `{"type":"m.room.message","room_id":"!other:x","sender":"@boss:x","content":{"body":"x"}}`, + pushrules.Context{MemberCount: 3}); rule != "@boss:x" || !out.Highlight { + t.Errorf("sender rule: %q %+v", rule, out) + } +} + +func TestMergeKeepsUserRulesAndDefaultOverrides(t *testing.T) { + defaults := pushrules.Defaults(alice) + stored := defaults.Clone() + if err := stored.SetEnabled(pushrules.KindOverride, pushrules.RuleMaster, true); err != nil { + t.Fatal(err) + } + if err := stored.SetActions(pushrules.KindUnderride, pushrules.RuleMessage, []json.RawMessage{}); err != nil { + t.Fatal(err) + } + mine := pushrules.Rule{RuleID: "mine", Enabled: true, Actions: []json.RawMessage{}, Conditions: []pushrules.Condition{}} + if err := stored.Put(pushrules.KindOverride, mine, "", ""); err != nil { + t.Fatal(err) + } + + // A default rule the stored copy does not have — one a newer server + // added — appears; one it has that the server dropped disappears. + stored.Override = slices.DeleteFunc(stored.Override, func(r pushrules.Rule) bool { return r.RuleID == pushrules.RuleTombstone }) + stored.Override = append(stored.Override, pushrules.Rule{RuleID: ".m.rule.gone", Default: true}) + + merged := pushrules.Merge(stored, defaults) + ids := ruleIDs(merged.Override) + if ids[0] != pushrules.RuleMaster || ids[1] != "mine" || !slices.Contains(ids, pushrules.RuleTombstone) || slices.Contains(ids, ".m.rule.gone") { + t.Errorf("override = %v", ids) + } + if master, _ := merged.Find(pushrules.KindOverride, pushrules.RuleMaster); !master.Enabled { + t.Error("the master rule lost its enabled flag") + } + if msg, _ := merged.Find(pushrules.KindUnderride, pushrules.RuleMessage); len(msg.Actions) != 0 { + t.Errorf("the message rule's actions = %s", msg.Actions) + } + + // The master rule silences everything once on. + if rule, out := evaluate(t, merged, `{"type":"m.room.message","content":{"body":"alice"}}`, pushrules.Context{}); rule != pushrules.RuleMaster || out.Notify { + t.Errorf("with master on: %q %+v", rule, out) + } +} + +func TestPutOrdering(t *testing.T) { + rs := pushrules.Defaults(alice) + mk := func(id string) pushrules.Rule { + return pushrules.Rule{RuleID: id, Enabled: true, Actions: []json.RawMessage{}, Conditions: []pushrules.Condition{}} + } + for _, id := range []string{"a", "b"} { + if err := rs.Put(pushrules.KindOverride, mk(id), "", ""); err != nil { + t.Fatal(err) + } + } + if err := rs.Put(pushrules.KindOverride, mk("c"), "", "a"); err != nil { + t.Fatal(err) + } + if err := rs.Put(pushrules.KindOverride, mk("d"), "b", ""); err != nil { + t.Fatal(err) + } + if got := ruleIDs(rs.Override)[:5]; !slices.Equal(got, []string{pushrules.RuleMaster, "d", "b", "a", "c"}) { + t.Errorf("order = %v", got) + } + + // Replacing keeps the place and the enabled flag. + if err := rs.SetEnabled(pushrules.KindOverride, "b", false); err != nil { + t.Fatal(err) + } + replacement := mk("b") + replacement.Actions = []json.RawMessage{json.RawMessage(`"notify"`)} + if err := rs.Put(pushrules.KindOverride, replacement, "", ""); err != nil { + t.Fatal(err) + } + b, _ := rs.Find(pushrules.KindOverride, "b") + if got := ruleIDs(rs.Override)[2]; got != "b" || b.Enabled || len(b.Actions) != 1 { + t.Errorf("after replacing: %v, %+v", ruleIDs(rs.Override), b) + } + + if err := rs.Put(pushrules.KindOverride, mk("e"), "nope", ""); !errors.Is(err, pushrules.ErrNoSuchAnchor) { + t.Errorf("before an unknown rule: %v", err) + } + if err := rs.Put(pushrules.KindOverride, mk("e"), pushrules.RuleMaster, ""); !errors.Is(err, pushrules.ErrNoSuchAnchor) { + t.Errorf("before a default rule: %v", err) + } + if err := rs.Delete(pushrules.KindOverride, pushrules.RuleMaster); !errors.Is(err, pushrules.ErrNoSuchRule) { + t.Errorf("deleting a default rule: %v", err) + } + if err := rs.Delete(pushrules.KindOverride, "a"); err != nil { + t.Errorf("deleting a user rule: %v", err) + } +} + +func TestBuildAndValidate(t *testing.T) { + notify := []json.RawMessage{json.RawMessage(`"notify"`)} + conds := []pushrules.Condition{} + pattern := "cake" + for _, tc := range []struct { + name string + kind pushrules.Kind + id string + rule pushrules.NewRule + ok bool + }{ + {"an override rule", pushrules.KindOverride, "x", pushrules.NewRule{Actions: notify, Conditions: &conds}, true}, + {"an override rule without conditions", pushrules.KindOverride, "x", pushrules.NewRule{Actions: notify}, false}, + {"a content rule", pushrules.KindContent, "x", pushrules.NewRule{Actions: notify, Pattern: &pattern}, true}, + {"a content rule without a pattern", pushrules.KindContent, "x", pushrules.NewRule{Actions: notify}, false}, + {"a room rule", pushrules.KindRoom, "!r:x", pushrules.NewRule{Actions: notify}, true}, + {"a reserved ID", pushrules.KindRoom, ".m.rule.x", pushrules.NewRule{Actions: notify}, false}, + {"no actions", pushrules.KindRoom, "!r:x", pushrules.NewRule{}, false}, + {"an unknown action", pushrules.KindRoom, "!r:x", pushrules.NewRule{Actions: []json.RawMessage{json.RawMessage(`"shout"`)}}, false}, + {"a legacy action", pushrules.KindRoom, "!r:x", pushrules.NewRule{Actions: []json.RawMessage{json.RawMessage(`"dont_notify"`)}}, true}, + {"a tweak", pushrules.KindRoom, "!r:x", pushrules.NewRule{Actions: []json.RawMessage{json.RawMessage(`{"set_tweak":"sound","value":"x"}`)}}, true}, + {"a tweak without a name", pushrules.KindRoom, "!r:x", pushrules.NewRule{Actions: []json.RawMessage{json.RawMessage(`{"value":"x"}`)}}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + rule, err := tc.rule.Build(tc.kind, tc.id) + if (err == nil) != tc.ok { + t.Fatalf("Build = %v", err) + } + if err != nil && !errors.Is(err, pushrules.ErrInvalidRule) { + t.Errorf("error %v is not ErrInvalidRule", err) + } + if err == nil && !rule.Enabled { + t.Error("a new rule is not enabled") + } + }) + } +} + +func TestParseActions(t *testing.T) { + out := pushrules.ParseActions([]json.RawMessage{ + json.RawMessage(`"notify"`), + json.RawMessage(`{"set_tweak":"sound","value":"default"}`), + json.RawMessage(`{"set_tweak":"highlight"}`), + json.RawMessage(`{"set_tweak":"org.example","value":{"a":1}}`), + }) + if !out.Notify || !out.Highlight || string(out.Tweaks["sound"]) != `"default"` || + string(out.Tweaks["highlight"]) != "true" || string(out.Tweaks["org.example"]) != `{"a":1}` { + t.Errorf("ParseActions = %+v", out) + } + if out := pushrules.ParseActions([]json.RawMessage{json.RawMessage(`{"set_tweak":"highlight","value":true}`)}); out.Notify || out.Highlight { + t.Errorf("a highlight that does not notify = %+v", out) + } + if out := pushrules.ParseActions([]json.RawMessage{json.RawMessage(`"dont_notify"`)}); out.Notify { + t.Errorf("dont_notify = %+v", out) + } +} + +func TestDocumentRoundTrip(t *testing.T) { + raw, err := pushrules.MarshalDocument(pushrules.Defaults(alice)) + if err != nil { + t.Fatal(err) + } + rs, err := pushrules.ParseDocument(raw) + if err != nil { + t.Fatal(err) + } + if len(rs.Override) != 12 || len(rs.Underride) != 5 { + t.Errorf("round trip lost rules: %s", raw) + } + if _, err := pushrules.ParseDocument([]byte(`nope`)); err == nil { + t.Error("a malformed document parsed") + } +} diff --git a/server/internal/pushrules/rules.go b/server/internal/pushrules/rules.go new file mode 100644 index 0000000..b057938 --- /dev/null +++ b/server/internal/pushrules/rules.go @@ -0,0 +1,273 @@ +// Package pushrules is the push rule language of the Client-Server API: +// the rule model, the server-default ruleset, and the evaluation that +// decides whether an event notifies a user. +// +// It is pure library code, like internal/matrix: no storage, no HTTP, no +// rooms. The caller supplies the event and the handful of facts about the +// room an evaluation needs — the user's display name in it, how many +// members it has, how much power the sender holds — and gets back the rule +// that matched. Where those facts come from, and what is done with the +// answer, is internal/pushserver's business. +package pushrules + +import ( + "encoding/json" + "fmt" + "slices" + "strings" +) + +// Kind is one of the five kinds of push rule, which are also the order +// they are evaluated in. +type Kind string + +// The kinds, as the spec spells them in paths and in a ruleset. +const ( + KindOverride Kind = "override" + KindContent Kind = "content" + KindRoom Kind = "room" + KindSender Kind = "sender" + KindUnderride Kind = "underride" +) + +// Kinds lists the kinds in evaluation order. +var Kinds = []Kind{KindOverride, KindContent, KindRoom, KindSender, KindUnderride} + +// ParseKind reads a kind from a path segment. +func ParseKind(s string) (Kind, bool) { + k := Kind(s) + return k, slices.Contains(Kinds, k) +} + +// Rule is one push rule, in the shape the Client-Server API reads and +// writes it. +type Rule struct { + RuleID string `json:"rule_id"` + Default bool `json:"default"` + Enabled bool `json:"enabled"` + + // Actions are kept as the JSON they arrived as. The spec has the + // server pass unknown tweaks through to the push gateway untouched, + // and a typed model would quietly drop them. + Actions []json.RawMessage `json:"actions"` + + // Conditions are only on override and underride rules. A room or + // sender rule's condition is its ID, and a content rule's is its + // pattern. + Conditions []Condition `json:"conditions,omitempty"` + + // Pattern is only on a content rule. + Pattern *string `json:"pattern,omitempty"` +} + +// MarshalJSON writes conditions whenever the rule has a list of them, an +// empty one included: .m.rule.master's "conditions": [] is how a client +// tells an override rule that always matches from a malformed one. +func (r Rule) MarshalJSON() ([]byte, error) { + type plain Rule + if r.Conditions == nil || len(r.Conditions) > 0 { + return json.Marshal(plain(r)) + } + return json.Marshal(struct { + plain + Conditions []Condition `json:"conditions"` + }{plain: plain(r), Conditions: []Condition{}}) +} + +// Condition is one condition of an override or underride rule. +type Condition struct { + Kind string `json:"kind"` + + // Key is the dotted path into the event for event_match, + // event_property_is and event_property_contains, and the notification + // key for sender_notification_permission. + Key string `json:"key,omitempty"` + + // Pattern is event_match's glob. + Pattern *string `json:"pattern,omitempty"` + + // Is is room_member_count's comparison, e.g. "2" or ">=10". + Is string `json:"is,omitempty"` + + // Value is what event_property_is and event_property_contains compare + // against. It is raw so that `null` is a value rather than an absence. + Value json.RawMessage `json:"value,omitempty"` +} + +// The condition kinds this server evaluates. A rule with any other kind of +// condition never matches, which is what the spec asks of a server that +// meets a condition it does not understand. +const ( + CondEventMatch = "event_match" + CondEventPropertyIs = "event_property_is" + CondEventPropertyContains = "event_property_contains" + CondContainsDisplayName = "contains_display_name" + CondRoomMemberCount = "room_member_count" + CondSenderNotificationPermission = "sender_notification_permission" +) + +// Ruleset is a user's rules, one list per kind, each in priority order. +type Ruleset struct { + Override []Rule `json:"override"` + Content []Rule `json:"content"` + Room []Rule `json:"room"` + Sender []Rule `json:"sender"` + Underride []Rule `json:"underride"` +} + +// Rules returns the list of one kind. +func (rs *Ruleset) Rules(kind Kind) []Rule { + switch kind { + case KindOverride: + return rs.Override + case KindContent: + return rs.Content + case KindRoom: + return rs.Room + case KindSender: + return rs.Sender + case KindUnderride: + return rs.Underride + } + return nil +} + +// SetRules replaces the list of one kind. +func (rs *Ruleset) SetRules(kind Kind, rules []Rule) { + switch kind { + case KindOverride: + rs.Override = rules + case KindContent: + rs.Content = rules + case KindRoom: + rs.Room = rules + case KindSender: + rs.Sender = rules + case KindUnderride: + rs.Underride = rules + } +} + +// Find returns a rule by kind and ID. +func (rs *Ruleset) Find(kind Kind, ruleID string) (Rule, bool) { + for _, r := range rs.Rules(kind) { + if r.RuleID == ruleID { + return r, true + } + } + return Rule{}, false +} + +// normalise replaces nil lists with empty ones, so that a ruleset always +// marshals with all five keys as arrays: a client reading `null` where it +// expects a list of rules is a client that crashes. +func (rs *Ruleset) normalise() { + for _, kind := range Kinds { + rules := rs.Rules(kind) + if rules == nil { + rules = []Rule{} + } + for i := range rules { + if rules[i].Actions == nil { + rules[i].Actions = []json.RawMessage{} + } + } + rs.SetRules(kind, rules) + } +} + +// MarshalJSON writes the ruleset with every kind present. +func (rs Ruleset) MarshalJSON() ([]byte, error) { + rs.normalise() + type plain Ruleset + return json.Marshal(plain(rs)) +} + +// Clone returns a deep enough copy that editing one list of it leaves the +// original alone. +func (rs Ruleset) Clone() Ruleset { + var out Ruleset + for _, kind := range Kinds { + out.SetRules(kind, slices.Clone(rs.Rules(kind))) + } + return out +} + +// IsReserved reports whether a rule ID is in the namespace the spec keeps +// for server-default rules. A client may change the actions of such a rule +// and switch it on and off, but may not create, replace or delete one. +func IsReserved(ruleID string) bool { return strings.HasPrefix(ruleID, ".") } + +// Merge lays a user's stored rules over the server defaults. +// +// What a user owns is the rules they made, and the enabled flag and +// actions of each default rule; the default rules themselves — their +// conditions, and which of them exist — are the server's, and come from +// defaults. Storing the whole ruleset and re-reading only those parts is +// what lets a server upgrade bring in a default rule a newer spec version +// added, without a migration rewriting every user's rules. +// +// The order within a kind is the spec's: user rules outrank the defaults +// of their kind, except that .m.rule.master stays ahead of everything so +// that switching it on still silences all. +func Merge(stored, defaults Ruleset) Ruleset { + var out Ruleset + for _, kind := range Kinds { + overrides := map[string]Rule{} + var user []Rule + for _, r := range stored.Rules(kind) { + if r.Default { + overrides[r.RuleID] = r + continue + } + if IsReserved(r.RuleID) { + // A user rule cannot have a reserved ID; one that does was + // not written by this server and is ignored rather than + // allowed to shadow a default. + continue + } + user = append(user, r) + } + + var prepend, appended []Rule + for _, d := range defaults.Rules(kind) { + if o, ok := overrides[d.RuleID]; ok { + d.Enabled = o.Enabled + d.Actions = o.Actions + } + if d.RuleID == RuleMaster { + prepend = append(prepend, d) + continue + } + appended = append(appended, d) + } + + rules := make([]Rule, 0, len(prepend)+len(user)+len(appended)) + rules = append(rules, prepend...) + rules = append(rules, user...) + rules = append(rules, appended...) + out.SetRules(kind, rules) + } + out.normalise() + return out +} + +// Document is the content of the m.push_rules account data: the ruleset +// under the one scope Matrix still has. +type Document struct { + Global Ruleset `json:"global"` +} + +// ParseDocument reads stored m.push_rules content. +func ParseDocument(raw []byte) (Ruleset, error) { + var doc Document + if err := json.Unmarshal(raw, &doc); err != nil { + return Ruleset{}, fmt.Errorf("pushrules: stored ruleset is not readable: %w", err) + } + return doc.Global, nil +} + +// MarshalDocument renders a ruleset as m.push_rules content. +func MarshalDocument(rs Ruleset) ([]byte, error) { + return json.Marshal(Document{Global: rs}) +} diff --git a/server/internal/pushserver/gateway.go b/server/internal/pushserver/gateway.go new file mode 100644 index 0000000..e2120b6 --- /dev/null +++ b/server/internal/pushserver/gateway.go @@ -0,0 +1,243 @@ +package pushserver + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "sync" + "time" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// The push gateway client: the Push Gateway API's one endpoint, +// POST /_matrix/push/v1/notify, called for every pusher of a notified user. +// +// Pushes are sent from a small pool of senders fed by a bounded queue, so +// that a slow or dead gateway delays its own pushes and nothing else: the +// evaluation worker never waits on the network. A push that fails with a +// server error or a timeout is retried a few times with a growing pause; +// one the gateway refuses outright is not. When the queue is full a push +// is dropped and logged. Pushes are a best-effort wake-up — the +// notification itself is already counted, and the device sees it on its +// next sync whether or not the push arrived. + +const ( + // gatewaySenders is how many pushes are in flight at once. + gatewaySenders = 4 + + // gatewayQueueSize bounds the pushes waiting for a sender. + gatewayQueueSize = 1024 + + // gatewayAttempts is how many times one push is tried. + gatewayAttempts = 3 + + // maxGatewayResponse bounds what is read of a gateway's answer, which + // is a list of rejected push keys and should be tiny. + maxGatewayResponse = 64 << 10 +) + +// gatewayRetryDelay is the pause before the given retry (1-based). +var gatewayRetryDelay = func(attempt int) time.Duration { + return time.Duration(1<<(2*(attempt-1))) * time.Second +} + +// notificationPush is the notification object of a push, without the +// devices, which are per pusher. +type notificationPush struct { + EventID string `json:"event_id,omitempty"` + RoomID string `json:"room_id,omitempty"` + Type string `json:"type,omitempty"` + Sender string `json:"sender,omitempty"` + SenderDisplayName string `json:"sender_display_name,omitempty"` + RoomName string `json:"room_name,omitempty"` + RoomAlias string `json:"room_alias,omitempty"` + UserIsTarget *bool `json:"user_is_target,omitempty"` + Membership string `json:"membership,omitempty"` + Prio string `json:"prio,omitempty"` + Content json.RawMessage `json:"content,omitempty"` + Counts pushCounts `json:"counts"` + Devices []pushDevice `json:"devices"` +} + +type pushCounts struct { + Unread int `json:"unread"` +} + +type pushDevice struct { + AppID string `json:"app_id"` + PushKey string `json:"pushkey"` + PushKeyTS int64 `json:"pushkey_ts,omitempty"` + Data map[string]json.RawMessage `json:"data,omitempty"` + Tweaks map[string]json.RawMessage `json:"tweaks,omitempty"` +} + +// pushJob is one push to one pusher. +type pushJob struct { + localpart string + pusher storage.Pusher + note notificationPush + tweaks map[string]json.RawMessage +} + +type gatewayOptions struct { + client *http.Client + includeContent bool + + // rejected is told about a push key the gateway refused. + rejected func(localpart, appID, pushKey string) + + log *slog.Logger +} + +type gateway struct { + opts gatewayOptions + queue chan pushJob + wg sync.WaitGroup + ctx context.Context +} + +func newGateway(opts gatewayOptions) *gateway { + return &gateway{opts: opts, queue: make(chan pushJob, gatewayQueueSize)} +} + +func (g *gateway) start(ctx context.Context) { + g.ctx = ctx + for range gatewaySenders { + g.wg.Add(1) + go g.send() + } +} + +func (g *gateway) wait() { g.wg.Wait() } + +func (g *gateway) enqueue(job pushJob) { + select { + case g.queue <- job: + default: + g.opts.log.Warn("push queue is full; dropping a push", + "localpart", job.localpart, "app_id", job.pusher.AppID) + } +} + +func (g *gateway) send() { + defer g.wg.Done() + for { + select { + case <-g.ctx.Done(): + return + case job := <-g.queue: + g.deliver(job) + } + } +} + +// deliver sends one push, retrying what is worth retrying. +func (g *gateway) deliver(job pushJob) { + var data pusherData + _ = json.Unmarshal(job.pusher.Data, &data) + body, err := g.payload(job, data) + if err != nil { + g.opts.log.Error("building a push failed", "localpart", job.localpart, "error", err) + return + } + + for attempt := 1; attempt <= gatewayAttempts; attempt++ { + rejected, retry, err := g.post(data.URL, body) + if err == nil { + for _, key := range rejected { + if key == job.pusher.PushKey { + g.opts.rejected(job.localpart, job.pusher.AppID, job.pusher.PushKey) + } + } + return + } + if !retry || attempt == gatewayAttempts { + g.opts.log.Warn("push gateway request failed", + "localpart", job.localpart, "app_id", job.pusher.AppID, "attempts", attempt, "error", err) + return + } + select { + case <-g.ctx.Done(): + return + case <-time.After(gatewayRetryDelay(attempt)): + } + } +} + +// payload builds the request body for one pusher. +// +// The pusher's data goes to the gateway as the device's data, less the +// URL, which is the homeserver's business. A pusher whose format is +// event_id_only gets only the IDs and the counts: the app fetches the +// event itself, so nothing of it passes through the gateway. +func (g *gateway) payload(job pushJob, data pusherData) ([]byte, error) { + deviceData := map[string]json.RawMessage{} + if err := json.Unmarshal(job.pusher.Data, &deviceData); err != nil { + return nil, fmt.Errorf("pusher data: %w", err) + } + delete(deviceData, "url") + + device := pushDevice{ + AppID: job.pusher.AppID, + PushKey: job.pusher.PushKey, + PushKeyTS: job.pusher.PushKeyTSMS / 1000, + Data: deviceData, + Tweaks: job.tweaks, + } + + note := job.note + if data.Format == "event_id_only" { + note = notificationPush{ + EventID: note.EventID, + RoomID: note.RoomID, + Prio: note.Prio, + Counts: note.Counts, + } + } else if !g.opts.includeContent { + note.Content = nil + } + note.Devices = []pushDevice{device} + return json.Marshal(struct { + Notification notificationPush `json:"notification"` + }{note}) +} + +// post sends a body to a gateway. It returns the push keys the gateway +// rejected, or whether a failure is worth another try. +func (g *gateway) post(url string, body []byte) (rejected []string, retry bool, err error) { + req, err := http.NewRequestWithContext(g.ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return nil, false, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := g.opts.client.Do(req) + if err != nil { + return nil, g.ctx.Err() == nil, err + } + defer func() { _ = resp.Body.Close() }() + + raw, err := io.ReadAll(io.LimitReader(resp.Body, maxGatewayResponse)) + if err != nil { + return nil, true, err + } + switch { + case resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500: + return nil, true, fmt.Errorf("gateway answered %d", resp.StatusCode) + case resp.StatusCode < 200 || resp.StatusCode >= 300: + return nil, false, fmt.Errorf("gateway answered %d", resp.StatusCode) + } + + var answer struct { + Rejected []string `json:"rejected"` + } + // A gateway that says 200 and sends something unreadable delivered + // the push all the same. + _ = json.Unmarshal(raw, &answer) + return answer.Rejected, false, nil +} diff --git a/server/internal/pushserver/gateway_test.go b/server/internal/pushserver/gateway_test.go new file mode 100644 index 0000000..49a1790 --- /dev/null +++ b/server/internal/pushserver/gateway_test.go @@ -0,0 +1,109 @@ +package pushserver + +import ( + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +func TestValidateGatewayURL(t *testing.T) { + for raw, ok := range map[string]bool{ + "https://push.example/_matrix/push/v1/notify": true, + "http://10.0.0.5:5000/_matrix/push/v1/notify": true, + "https://push.example/notify": false, + "ftp://push.example/_matrix/push/v1/notify": false, + "/_matrix/push/v1/notify": false, + "https://user:pw@push.example/_matrix/push/v1/notify": false, + } { + err := validateGatewayURL(raw) + if (err == nil) != ok { + t.Errorf("validateGatewayURL(%q) = %v", raw, err) + } + if err != nil && !errors.Is(err, ErrInvalidPusher) { + t.Errorf("validateGatewayURL(%q) = %v, not ErrInvalidPusher", raw, err) + } + } +} + +func TestGatewayRetriesServerErrorsAndNotRefusals(t *testing.T) { + saved := gatewayRetryDelay + gatewayRetryDelay = func(int) time.Duration { return time.Millisecond } + t.Cleanup(func() { gatewayRetryDelay = saved }) + + var calls atomic.Int32 + status := atomic.Int32{} + status.Store(http.StatusInternalServerError) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + _, _ = io.Copy(io.Discard, r.Body) + w.WriteHeader(int(status.Load())) + _, _ = w.Write([]byte(`{"rejected":[]}`)) + })) + defer ts.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + g := newGateway(gatewayOptions{ + client: ts.Client(), + rejected: func(string, string, string) {}, + log: slog.New(slog.DiscardHandler), + }) + g.ctx = ctx + + job := pushJob{ + localpart: "alice", + pusher: storage.Pusher{AppID: "a", PushKey: "k", + Data: json.RawMessage(`{"url":"` + ts.URL + `/_matrix/push/v1/notify"}`)}, + note: notificationPush{EventID: "$e", RoomID: "!r:x"}, + } + + g.deliver(job) + if got := calls.Load(); got != gatewayAttempts { + t.Errorf("a failing gateway was tried %d times, want %d", got, gatewayAttempts) + } + + calls.Store(0) + status.Store(http.StatusBadRequest) + g.deliver(job) + if got := calls.Load(); got != 1 { + t.Errorf("a refusing gateway was tried %d times, want 1", got) + } +} + +func TestGatewayPayloadLeavesTheURLOut(t *testing.T) { + g := newGateway(gatewayOptions{includeContent: false, log: slog.New(slog.DiscardHandler)}) + job := pushJob{ + pusher: storage.Pusher{AppID: "a", PushKey: "k", PushKeyTSMS: 5000, + Data: json.RawMessage(`{"url":"https://p.example/_matrix/push/v1/notify","extra":1}`)}, + note: notificationPush{EventID: "$e", Content: json.RawMessage(`{"body":"secret"}`)}, + } + raw, err := g.payload(job, pusherData{}) + if err != nil { + t.Fatal(err) + } + var body struct { + Notification struct { + Content json.RawMessage `json:"content"` + Devices []struct { + PushKeyTS int64 `json:"pushkey_ts"` + Data map[string]any `json:"data"` + } `json:"devices"` + } `json:"notification"` + } + if err := json.Unmarshal(raw, &body); err != nil { + t.Fatal(err) + } + d := body.Notification.Devices[0] + if body.Notification.Content != nil || d.Data["url"] != nil || d.Data["extra"] != float64(1) || d.PushKeyTS != 5 { + t.Errorf("payload = %s", raw) + } +} diff --git a/server/internal/pushserver/notifications.go b/server/internal/pushserver/notifications.go new file mode 100644 index 0000000..558f42c --- /dev/null +++ b/server/internal/pushserver/notifications.go @@ -0,0 +1,162 @@ +package pushserver + +import ( + "context" + "encoding/json" + "errors" + "strconv" + + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Receipt types that move the counts. A private receipt is still the +// user saying they have read up to here; it is private from other users, +// not from the user's own badge. +const ( + receiptRead = "m.read" + receiptReadPrivate = "m.read.private" + + // threadMain is the thread ID of a receipt for the main timeline. + threadMain = "main" +) + +// ReceiptSet is told about a read receipt a local user set, and clears +// their notifications in the room up to the event it names. +// +// Notifications are not tracked per thread (threads are Phase 10), so a +// receipt counts when it is for the room as a whole or for its main +// timeline, and a receipt inside a thread is left alone: clearing the +// room on it would mark the main timeline read when the user only looked +// at one thread. Thread notifications are cleared when the main receipt +// moves past them. +func (a *API) ReceiptSet(ctx context.Context, userID, roomID, receiptType, threadID, eventID string) { + if receiptType != receiptRead && receiptType != receiptReadPrivate { + return + } + if threadID != "" && threadID != threadMain { + return + } + pos, err := a.rooms.EventPosition(ctx, roomID, userID, eventID) + if err != nil { + // A receipt for an event this server does not have — or one the + // user cannot see — reads nothing. + if !errors.Is(err, roomserver.ErrNoSuchEvent) && !errors.Is(err, roomserver.ErrForbidden) { + a.log.Warn("locating a receipt's event failed", + "user_id", userID, "room_id", roomID, "event_id", eventID, "error", err) + } + return + } + if err := a.markRead(ctx, userID, roomID, pos); err != nil { + a.log.Warn("clearing notifications on a receipt failed", + "user_id", userID, "room_id", roomID, "error", err) + } +} + +// markRead clears a user's notifications in a room up to a position, and +// wakes their syncs if the counts moved. +func (a *API) markRead(ctx context.Context, userID, roomID string, upTo int64) error { + _, moved, err := a.store.MarkNotificationsRead(ctx, userID, roomID, upTo) + if err != nil { + return err + } + if moved { + a.notifyUsers(userID) + } + return nil +} + +// NotificationCounts reads a user's unread counts in the given rooms. +func (a *API) NotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]storage.NotificationCounts, error) { + return a.store.NotificationCounts(ctx, userID, roomIDs) +} + +// NotificationCountChanges lists the rooms whose counts for a user moved +// in a window of the notification stream. +func (a *API) NotificationCountChanges(ctx context.Context, userID string, from, to int64) ([]string, error) { + return a.store.NotificationCountChanges(ctx, userID, from, to) +} + +// Notification is one entry of GET /notifications. +type Notification struct { + Actions []json.RawMessage `json:"actions"` + Event roomserver.ClientEvent `json:"event"` + ProfileTag string `json:"profile_tag,omitempty"` + Read bool `json:"read"` + RoomID string `json:"room_id"` + TS int64 `json:"ts"` +} + +// NotificationsRequest is GET /notifications. +type NotificationsRequest struct { + UserID string + + // From is the next_token of an earlier page, or empty for the newest. + From string + + Limit int + OnlyHighlight bool +} + +// Limits on a page of notifications. +const ( + DefaultNotificationsLimit = 20 + MaxNotificationsLimit = 100 +) + +// ErrInvalidFrom is a from token that is not one this server handed out. +var ErrInvalidFrom = errors.New("invalid pagination token") + +// Notifications pages through a user's notifications, newest first. It +// returns the page and the token for the next one, which is empty at the +// end. +func (a *API) Notifications(ctx context.Context, req NotificationsRequest) ([]Notification, string, error) { + var before int64 + if req.From != "" { + var err error + if before, err = strconv.ParseInt(req.From, 10, 64); err != nil || before <= 0 { + return nil, "", ErrInvalidFrom + } + } + limit := req.Limit + if limit <= 0 { + limit = DefaultNotificationsLimit + } + limit = min(limit, MaxNotificationsLimit) + + stored, err := a.store.Notifications(ctx, req.UserID, before, limit, req.OnlyHighlight) + if err != nil { + return nil, "", err + } + + out := make([]Notification, 0, len(stored)) + for _, n := range stored { + ev, err := a.rooms.Event(ctx, n.RoomID, req.UserID, n.EventID) + switch { + case errors.Is(err, roomserver.ErrNoSuchEvent), errors.Is(err, roomserver.ErrNoSuchRoom), + errors.Is(err, roomserver.ErrForbidden): + // Purged, or no longer the user's to see: the notification + // has nothing left to show. + continue + case err != nil: + return nil, "", err + } + var actions []json.RawMessage + if err := json.Unmarshal(n.Actions, &actions); err != nil { + actions = []json.RawMessage{} + } + out = append(out, Notification{ + Actions: actions, + Event: ev, + Read: n.Read, + RoomID: n.RoomID, + TS: n.TimestampMS, + }) + } + + var next string + if len(stored) == limit { + next = strconv.FormatInt(stored[len(stored)-1].ID, 10) + } + return out, next, nil +} diff --git a/server/internal/pushserver/pushers.go b/server/internal/pushserver/pushers.go new file mode 100644 index 0000000..355c8e4 --- /dev/null +++ b/server/internal/pushserver/pushers.go @@ -0,0 +1,179 @@ +package pushserver + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// PusherKindHTTP is the one kind of pusher this server pushes to. The +// spec's other kind, email, needs a mail server and templates, which are +// not something a homeserver should ship by default; a client asking for +// it is told so rather than given a pusher that never fires. +const PusherKindHTTP = "http" + +// gatewayPath is the path every push gateway is reached at. +const gatewayPath = "/_matrix/push/v1/notify" + +// Limits the spec sets on a pusher's identity. +const ( + maxAppIDLength = 64 + maxPushKeyLength = 512 +) + +// Pusher is a pusher in the shape the Client-Server API uses for it. +type Pusher struct { + PushKey string `json:"pushkey"` + Kind string `json:"kind"` + AppID string `json:"app_id"` + AppDisplayName string `json:"app_display_name"` + DeviceDisplayName string `json:"device_display_name"` + ProfileTag string `json:"profile_tag,omitempty"` + Lang string `json:"lang"` + Data json.RawMessage `json:"data"` +} + +// SetPusherRequest is POST /pushers/set. +type SetPusherRequest struct { + PushKey string `json:"pushkey"` + + // Kind null removes the pusher. + Kind *string `json:"kind"` + + AppID string `json:"app_id"` + AppDisplayName string `json:"app_display_name"` + DeviceDisplayName string `json:"device_display_name"` + ProfileTag string `json:"profile_tag"` + Lang string `json:"lang"` + Data json.RawMessage `json:"data"` + Append bool `json:"append"` +} + +// pusherData is the part of a pusher's data the server reads. +type pusherData struct { + URL string `json:"url"` + Format string `json:"format"` +} + +// Pushers lists a user's pushers. +func (a *API) Pushers(ctx context.Context, localpart string) ([]Pusher, error) { + stored, err := a.store.Pushers(ctx, localpart) + if err != nil { + return nil, err + } + out := make([]Pusher, 0, len(stored)) + for _, p := range stored { + out = append(out, Pusher{ + PushKey: p.PushKey, Kind: p.Kind, AppID: p.AppID, + AppDisplayName: p.AppDisplayName, DeviceDisplayName: p.DeviceDisplayName, + ProfileTag: p.ProfileTag, Lang: p.Lang, Data: p.Data, + }) + } + return out, nil +} + +// SetPusher creates, replaces or removes a pusher for the session +// deviceID. +func (a *API) SetPusher(ctx context.Context, localpart, deviceID string, req SetPusherRequest) error { + switch { + case req.PushKey == "": + return fmt.Errorf("%w: pushkey", ErrMissingPusherField) + case req.AppID == "": + return fmt.Errorf("%w: app_id", ErrMissingPusherField) + case len(req.AppID) > maxAppIDLength: + return fmt.Errorf("%w: app_id must be at most %d characters", ErrInvalidPusher, maxAppIDLength) + case len(req.PushKey) > maxPushKeyLength: + return fmt.Errorf("%w: pushkey must be at most %d bytes", ErrInvalidPusher, maxPushKeyLength) + } + + if req.Kind == nil { + return a.store.DeletePusher(ctx, localpart, req.AppID, req.PushKey) + } + + switch { + case *req.Kind != PusherKindHTTP: + return fmt.Errorf("%w: this server only supports pushers of kind %q", ErrInvalidPusher, PusherKindHTTP) + case req.AppDisplayName == "": + return fmt.Errorf("%w: app_display_name", ErrMissingPusherField) + case req.DeviceDisplayName == "": + return fmt.Errorf("%w: device_display_name", ErrMissingPusherField) + case req.Lang == "": + return fmt.Errorf("%w: lang", ErrMissingPusherField) + case len(req.Data) == 0: + return fmt.Errorf("%w: data", ErrMissingPusherField) + } + + var data pusherData + if err := json.Unmarshal(req.Data, &data); err != nil { + return fmt.Errorf("%w: data must be an object", ErrInvalidPusher) + } + if data.URL == "" { + return fmt.Errorf("%w: data.url", ErrMissingPusherField) + } + if err := validateGatewayURL(data.URL); err != nil { + return err + } + + return a.store.SetPusher(ctx, storage.Pusher{ + Localpart: localpart, + DeviceID: deviceID, + Kind: *req.Kind, + AppID: req.AppID, + PushKey: req.PushKey, + AppDisplayName: req.AppDisplayName, + DeviceDisplayName: req.DeviceDisplayName, + ProfileTag: req.ProfileTag, + Lang: req.Lang, + Data: req.Data, + PushKeyTSMS: a.now().UnixMilli(), + }, req.Append) +} + +// validateGatewayURL checks a pusher's gateway URL. The spec asks for +// HTTPS; plain HTTP is accepted too, because the usual deployment runs the +// gateway next to the homeserver on a private network, where requiring a +// certificate would only push operators to a worse workaround. +func validateGatewayURL(raw string) error { + u, err := url.Parse(raw) + switch { + case err != nil || u.Host == "": + return fmt.Errorf("%w: data.url must be an absolute URL", ErrInvalidPusher) + case u.Scheme != "https" && u.Scheme != "http": + return fmt.Errorf("%w: data.url must be an http or https URL", ErrInvalidPusher) + case u.Path != gatewayPath: + return fmt.Errorf("%w: data.url must have the path %s", ErrInvalidPusher, gatewayPath) + case u.User != nil: + return fmt.Errorf("%w: data.url must not carry credentials", ErrInvalidPusher) + } + return nil +} + +// DevicesDeleted removes the pushers of deleted sessions: a phone that +// has logged out must stop being woken for the account. It implements +// userapi.DeviceObserver, and like every observer it logs a failure +// rather than returning it, because the device is already gone. +func (a *API) DevicesDeleted(ctx context.Context, localpart string, deviceIDs []string) { + if err := a.store.DeletePushersForDevices(ctx, localpart, deviceIDs); err != nil { + a.log.Warn("removing the pushers of deleted devices failed", + "localpart", localpart, "devices", deviceIDs, "error", err) + } +} + +// DeviceRenamed implements userapi.DeviceObserver. A pusher's device +// display name is the one the app registered it with, not the session's. +func (a *API) DeviceRenamed(context.Context, string, string) {} + +// pushKeyRejected removes a pusher whose gateway said its push key is no +// longer valid — the app was uninstalled, or its registration expired — +// so that it is not pushed to again. +func (a *API) pushKeyRejected(localpart, appID, pushKey string) { + if err := a.store.DeletePusher(a.ctx, localpart, appID, pushKey); err != nil && a.ctx.Err() == nil { + a.log.Warn("removing a rejected pusher failed", + "localpart", localpart, "app_id", appID, "error", err) + return + } + a.log.Info("removed a pusher its gateway rejected", "localpart", localpart, "app_id", appID) +} diff --git a/server/internal/pushserver/pushserver.go b/server/internal/pushserver/pushserver.go new file mode 100644 index 0000000..0b6277c --- /dev/null +++ b/server/internal/pushserver/pushserver.go @@ -0,0 +1,251 @@ +// Package pushserver turns events into notifications: it keeps each user's +// push rules and pushers, evaluates every event against the rules of every +// local member of its room, records what notified whom with per-room unread +// counts, and pushes to the push gateways of the users' devices. +// +// It is PLAN.md Phase 8. The rule language is internal/pushrules; this +// package is everything around it that needs storage, rooms or a network. +// +// It hangs off the roomserver's emit stage like the federation queue does, +// and like the queue it does no work there: the emit stage runs with the +// room's write lock held, and evaluating an event needs the room's members, +// their display names, the power levels and a ruleset per member. So an +// event is put on a list and a worker of this package's own takes it from +// there. What that costs is a moment in which a sync can show a message +// whose notification has not been counted yet; the counts are a stream of +// their own in the sync token (see storage.PushStore), so the sync that +// follows the worker's write carries the corrected count. +package pushserver + +import ( + "context" + "errors" + "log/slog" + "net/http" + "sync" + "time" + + "github.com/dariuszjaniak/warp/server/internal/pushrules" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Errors callers have to tell apart. The rule-language sentinels are +// re-exported so that a handler needs only this package. +var ( + ErrNoSuchRule = pushrules.ErrNoSuchRule + ErrNoSuchAnchor = pushrules.ErrNoSuchAnchor + ErrInvalidRule = pushrules.ErrInvalidRule + + // ErrInvalidPusher is a pusher a client may not register. + ErrInvalidPusher = errors.New("invalid pusher") + + // ErrMissingPusherField is a pusher missing a field the spec requires. + ErrMissingPusherField = errors.New("missing pusher field") +) + +// Accounts is the part of the userapi this package needs: push rules are +// the m.push_rules account data. *userapi.API satisfies it. +type Accounts interface { + AccountData(ctx context.Context, localpart, roomID, dataType string) ([]byte, error) + SetAccountData(ctx context.Context, localpart, roomID, dataType string, content []byte) error +} + +// Rooms is the part of the roomserver this package reads. *roomserver.API +// satisfies it. +type Rooms interface { + PushRoomFor(ctx context.Context, roomID string) (roomserver.PushRoom, error) + Event(ctx context.Context, roomID, userID, eventID string) (roomserver.ClientEvent, error) + EventPosition(ctx context.Context, roomID, userID, eventID string) (int64, error) +} + +// Notifier wakes a user's blocked syncs. *syncapi.Notifier satisfies it. +type Notifier interface { + NotifyUser(userID string) + NotifyUsers(userIDs []string) +} + +// API is the push server. +type API struct { + serverName string + store storage.PushStore + accounts Accounts + rooms Rooms + notifier Notifier + gateway *gateway + log *slog.Logger + now func() time.Time + + // pushEnabled sends pushes to gateways. Notifications are counted + // either way: the counts are what a client shows, and are no less + // right on a server that does not push. + pushEnabled bool + + // retention is how long a read notification is kept for GET + // /notifications before it is pruned. + retention time.Duration + + // rules caches each user's merged ruleset. Every event is evaluated + // against the rules of every local member of its room, and reading + // and merging them from account data each time would put a query per + // member in front of every message. Every write goes through this + // package, which is what makes the cache safe in one process. + rulesMu sync.Mutex + rulesCache map[string]pushrules.Ruleset + writeLocks sync.Map // localpart -> *sync.Mutex + + // The event list the emit stage appends to and the worker drains. + mu sync.Mutex + incoming []roomserver.Notification + busy bool + wake chan struct{} + idle *sync.Cond + + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup +} + +// Options configure the component. +type Options struct { + ServerName string + Store storage.PushStore + Accounts Accounts + Rooms Rooms + + // Notifier wakes the syncs a count change concerns. Nil wakes + // nobody, which only a test wants. + Notifier Notifier + + // PushEnabled sends pushes to the pushers' gateways. + PushEnabled bool + + // IncludeContent puts the event's content into a push. Off, a push + // carries only who sent what kind of event where, for operators who + // would rather not have message text pass through a third-party push + // service. + IncludeContent bool + + // GatewayTimeout bounds one request to a push gateway. + GatewayTimeout time.Duration + + // HTTPClient sends pushes; nil builds one. Tests pass their own. + HTTPClient *http.Client + + // NotificationRetention is how long a read notification is kept. + NotificationRetention time.Duration + + Logger *slog.Logger + Now func() time.Time +} + +// Defaults. +const ( + DefaultGatewayTimeout = 10 * time.Second + DefaultNotificationRetention = 30 * 24 * time.Hour + + // pruneInterval is how often read notifications past their retention + // are deleted. + pruneInterval = time.Hour +) + +// New builds the component and starts its worker. Close stops it. +func New(opts Options) (*API, error) { + switch { + case opts.ServerName == "": + return nil, errors.New("pushserver: ServerName must not be empty") + case opts.Store == nil: + return nil, errors.New("pushserver: Store must not be nil") + case opts.Accounts == nil: + return nil, errors.New("pushserver: Accounts must not be nil") + case opts.Rooms == nil: + return nil, errors.New("pushserver: Rooms must not be nil") + } + if opts.Logger == nil { + opts.Logger = slog.Default() + } + if opts.Now == nil { + opts.Now = time.Now + } + if opts.GatewayTimeout <= 0 { + opts.GatewayTimeout = DefaultGatewayTimeout + } + if opts.NotificationRetention <= 0 { + opts.NotificationRetention = DefaultNotificationRetention + } + if opts.HTTPClient == nil { + opts.HTTPClient = &http.Client{Timeout: opts.GatewayTimeout} + } + + ctx, cancel := context.WithCancel(context.Background()) + a := &API{ + serverName: opts.ServerName, + store: opts.Store, + accounts: opts.Accounts, + rooms: opts.Rooms, + notifier: opts.Notifier, + log: opts.Logger, + now: opts.Now, + pushEnabled: opts.PushEnabled, + retention: opts.NotificationRetention, + rulesCache: map[string]pushrules.Ruleset{}, + wake: make(chan struct{}, 1), + ctx: ctx, + cancel: cancel, + } + a.idle = sync.NewCond(&a.mu) + a.gateway = newGateway(gatewayOptions{ + client: opts.HTTPClient, + includeContent: opts.IncludeContent, + rejected: a.pushKeyRejected, + log: opts.Logger, + }) + + a.wg.Add(2) + go a.work() + go a.prune() + a.gateway.start(ctx) + return a, nil +} + +// Close stops the worker and the gateway senders. Events not yet +// evaluated are dropped: the emit stage is not durable, and a +// notification is not worth holding up a shutdown for. +func (a *API) Close() { + a.cancel() + a.mu.Lock() + a.idle.Broadcast() + a.mu.Unlock() + a.wg.Wait() + a.gateway.wait() +} + +// notifyUsers wakes syncs, if there is anybody to wake them. +func (a *API) notifyUsers(userIDs ...string) { + if a.notifier == nil || len(userIDs) == 0 { + return + } + a.notifier.NotifyUsers(userIDs) +} + +// prune deletes read notifications past their retention, hourly. +func (a *API) prune() { + defer a.wg.Done() + ticker := time.NewTicker(pruneInterval) + defer ticker.Stop() + for { + select { + case <-a.ctx.Done(): + return + case <-ticker.C: + } + cutoff := a.now().Add(-a.retention).UnixMilli() + n, err := a.store.PruneNotifications(a.ctx, cutoff) + switch { + case err != nil && a.ctx.Err() == nil: + a.log.Warn("pruning read notifications failed", "error", err) + case n > 0: + a.log.Debug("pruned read notifications", "count", n) + } + } +} diff --git a/server/internal/pushserver/rules.go b/server/internal/pushserver/rules.go new file mode 100644 index 0000000..d911770 --- /dev/null +++ b/server/internal/pushserver/rules.go @@ -0,0 +1,167 @@ +package pushserver + +import ( + "context" + "encoding/json" + "errors" + "sync" + + "github.com/dariuszjaniak/warp/server/internal/pushrules" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// AccountDataType is the account data push rules are kept in, and the +// type a client is refused when it tries to write them directly. +const AccountDataType = "m.push_rules" + +// Push rules are the m.push_rules account data, and this is the only +// writer of it. +// +// That is where the spec has a client read them — every client takes its +// rules from the account data of its initial sync — so keeping them +// anywhere else would mean a second copy, rendered into account data on +// every change. What is stored is the whole ruleset as a client sees it. +// What is read back from it is only what the user owns: their own rules, +// and the enabled flag and actions of each default rule. The rest comes +// from pushrules.Defaults, so a server upgrade that adds a default rule +// reaches every user without rewriting anybody's account data. + +// Ruleset returns a user's rules as they apply now: their own laid over +// the server defaults. +func (a *API) Ruleset(ctx context.Context, localpart, userID string) (pushrules.Ruleset, error) { + a.rulesMu.Lock() + cached, ok := a.rulesCache[localpart] + a.rulesMu.Unlock() + if ok { + return cached.Clone(), nil + } + + rs, err := a.loadRuleset(ctx, localpart, userID) + if err != nil { + return pushrules.Ruleset{}, err + } + a.rulesMu.Lock() + a.rulesCache[localpart] = rs + a.rulesMu.Unlock() + return rs.Clone(), nil +} + +func (a *API) loadRuleset(ctx context.Context, localpart, userID string) (pushrules.Ruleset, error) { + defaults := pushrules.Defaults(userID) + raw, err := a.accounts.AccountData(ctx, localpart, "", AccountDataType) + switch { + case errors.Is(err, storage.ErrNoSuchAccountData): + return defaults, nil + case err != nil: + return pushrules.Ruleset{}, err + } + stored, err := pushrules.ParseDocument(raw) + if err != nil { + // Only this package writes the rules, so an unreadable copy is a + // bug or a hand edit. The defaults are a better answer than + // refusing every push and every sync until somebody fixes it. + a.log.Error("stored push rules are unreadable; using the defaults", + "user_id", userID, "error", err) + return defaults, nil + } + return pushrules.Merge(stored, defaults), nil +} + +// Document is a user's rules as m.push_rules content: what a sync and a +// read of the account data return. +func (a *API) Document(ctx context.Context, localpart, userID string) ([]byte, error) { + rs, err := a.Ruleset(ctx, localpart, userID) + if err != nil { + return nil, err + } + return pushrules.MarshalDocument(rs) +} + +// Rule returns one rule. +func (a *API) Rule(ctx context.Context, localpart, userID string, kind pushrules.Kind, ruleID string) (pushrules.Rule, error) { + rs, err := a.Ruleset(ctx, localpart, userID) + if err != nil { + return pushrules.Rule{}, err + } + rule, ok := rs.Find(kind, ruleID) + if !ok { + return pushrules.Rule{}, ErrNoSuchRule + } + return rule, nil +} + +// PutRule creates or replaces a user rule; see pushrules.Ruleset.Put for +// where it goes. +func (a *API) PutRule(ctx context.Context, localpart, userID string, kind pushrules.Kind, ruleID string, rule pushrules.NewRule, before, after string) error { + built, err := rule.Build(kind, ruleID) + if err != nil { + return err + } + return a.editRules(ctx, localpart, userID, func(rs *pushrules.Ruleset) error { + return rs.Put(kind, built, before, after) + }) +} + +// DeleteRule removes a user rule. +func (a *API) DeleteRule(ctx context.Context, localpart, userID string, kind pushrules.Kind, ruleID string) error { + return a.editRules(ctx, localpart, userID, func(rs *pushrules.Ruleset) error { + return rs.Delete(kind, ruleID) + }) +} + +// SetRuleEnabled switches a rule on or off. +func (a *API) SetRuleEnabled(ctx context.Context, localpart, userID string, kind pushrules.Kind, ruleID string, enabled bool) error { + return a.editRules(ctx, localpart, userID, func(rs *pushrules.Ruleset) error { + return rs.SetEnabled(kind, ruleID, enabled) + }) +} + +// SetRuleActions replaces a rule's actions. +func (a *API) SetRuleActions(ctx context.Context, localpart, userID string, kind pushrules.Kind, ruleID string, actions []json.RawMessage) error { + return a.editRules(ctx, localpart, userID, func(rs *pushrules.Ruleset) error { + return rs.SetActions(kind, ruleID, actions) + }) +} + +// editRules is the read-modify-write every change is, serialised per user +// so that two changes made at once cannot each start from the ruleset the +// other has not written yet. +func (a *API) editRules(ctx context.Context, localpart, userID string, edit func(*pushrules.Ruleset) error) error { + mu := a.writeLock(localpart) + mu.Lock() + defer mu.Unlock() + + rs, err := a.Ruleset(ctx, localpart, userID) + if err != nil { + return err + } + if err := edit(&rs); err != nil { + return err + } + doc, err := pushrules.MarshalDocument(rs) + if err != nil { + return err + } + if err := a.accounts.SetAccountData(ctx, localpart, "", AccountDataType, doc); err != nil { + return err + } + + a.rulesMu.Lock() + a.rulesCache[localpart] = rs + a.rulesMu.Unlock() + + // The rules travel as account data, so a change is something every + // one of the user's syncs has to hear about now rather than at its + // timeout. + a.notifyUsers(userID) + return nil +} + +func (a *API) writeLock(localpart string) *sync.Mutex { + actual, _ := a.writeLocks.LoadOrStore(localpart, &sync.Mutex{}) + mu, ok := actual.(*sync.Mutex) + if !ok { + panic("pushserver: write lock map holds something that is not a mutex") + } + return mu +} diff --git a/server/internal/pushserver/worker.go b/server/internal/pushserver/worker.go new file mode 100644 index 0000000..5a95ae1 --- /dev/null +++ b/server/internal/pushserver/worker.go @@ -0,0 +1,278 @@ +package pushserver + +import ( + "context" + "encoding/json" + + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" + "github.com/dariuszjaniak/warp/server/internal/pushrules" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// EventPersisted implements roomserver.Notifier. It only queues the event: +// see the package comment for why nothing is evaluated here. +func (a *API) EventPersisted(_ context.Context, n roomserver.Notification) { + if n.Event == nil { + return + } + a.mu.Lock() + if a.ctx.Err() != nil { + a.mu.Unlock() + return + } + a.incoming = append(a.incoming, n) + a.mu.Unlock() + select { + case a.wake <- struct{}{}: + default: + } +} + +// WaitIdle blocks until every event queued so far has been evaluated, or +// ctx ends. Tests use it to look at counts without sleeping, and it is the +// honest way for anything else to ask "has the push server caught up". +func (a *API) WaitIdle(ctx context.Context) error { + stop := context.AfterFunc(ctx, func() { + a.mu.Lock() + a.idle.Broadcast() + a.mu.Unlock() + }) + defer stop() + + a.mu.Lock() + defer a.mu.Unlock() + for len(a.incoming) > 0 || a.busy { + if err := ctx.Err(); err != nil { + return err + } + if a.ctx.Err() != nil { + return a.ctx.Err() + } + a.idle.Wait() + } + return nil +} + +// work drains the event list, one event at a time and in the order they +// were persisted, so that a user's notifications are numbered in the +// order their events happened. +func (a *API) work() { + defer a.wg.Done() + for { + select { + case <-a.ctx.Done(): + return + case <-a.wake: + } + for { + a.mu.Lock() + if len(a.incoming) == 0 { + a.busy = false + a.idle.Broadcast() + a.mu.Unlock() + break + } + batch := a.incoming + a.incoming = nil + a.busy = true + a.mu.Unlock() + + for _, n := range batch { + if a.ctx.Err() != nil { + return + } + if err := a.process(a.ctx, n); err != nil && a.ctx.Err() == nil { + a.log.Error("evaluating push rules failed", + "room_id", n.RoomID, "event_id", n.Event.EventID(), "error", err) + } + } + } + } +} + +// process evaluates one event for every local user it may concern. +func (a *API) process(ctx context.Context, n roomserver.Notification) error { + e := n.Event + sender := e.Sender() + + // A local user leaving a room has nothing more to read in it, and a + // count left behind there would sit in their badge for good. The event + // is still evaluated for everybody else below. + if e.Type() == event.TypeMember { + if target, ok := e.StateKey(); ok && a.isLocal(target) { + switch membership, _ := e.Membership(); membership { + case event.MembershipLeave, event.MembershipBan: + if err := a.markRead(ctx, target, n.RoomID, n.StreamOrdering); err != nil { + return err + } + } + } + } + + room, err := a.rooms.PushRoomFor(ctx, n.RoomID) + if err != nil { + return err + } + + // Who the event is for: the room's local members, and the user an + // invite is addressed to, who is not a member yet. Never the sender — + // nobody is notified of what they did themselves. + var candidates []string + for userID := range room.Members { + if userID != sender && a.isLocal(userID) { + candidates = append(candidates, userID) + } + } + if e.Type() == event.TypeMember { + if target, ok := e.StateKey(); ok && target != sender && a.isLocal(target) { + if _, member := room.Members[target]; !member { + candidates = append(candidates, target) + } + } + } + if len(candidates) == 0 { + return nil + } + + ev, err := pushrules.NewEvent(e.JSON()) + if err != nil { + return err + } + + var ( + notifications []storage.NewNotification + outcomes = map[string]pushrules.Outcome{} + ) + now := a.now().UnixMilli() + for _, userID := range candidates { + localpart, _, err := id.SplitUserID(userID) + if err != nil { + continue + } + rs, err := a.Ruleset(ctx, localpart, userID) + if err != nil { + return err + } + rule, _, ok := rs.Evaluate(ev, pushrules.Context{ + UserID: userID, + DisplayName: room.Members[userID], + MemberCount: len(room.Members), + SenderMayNotify: func(key string) bool { + return room.SenderMayNotify(sender, key) + }, + }) + if !ok { + continue + } + outcome := pushrules.ParseActions(rule.Actions) + if !outcome.Notify { + continue + } + actions, err := json.Marshal(rule.Actions) + if err != nil { + return err + } + notifications = append(notifications, storage.NewNotification{ + UserID: userID, + RoomID: n.RoomID, + EventID: e.EventID(), + EventPos: n.StreamOrdering, + TimestampMS: now, + Highlight: outcome.Highlight, + Actions: actions, + }) + outcomes[userID] = outcome + } + if len(notifications) == 0 { + return nil + } + + if _, err := a.store.AddNotifications(ctx, notifications); err != nil { + return err + } + notified := make([]string, 0, len(notifications)) + for _, nn := range notifications { + notified = append(notified, nn.UserID) + } + a.notifyUsers(notified...) + + if a.pushEnabled { + for _, userID := range notified { + a.push(ctx, userID, room, n, outcomes[userID]) + } + } + return nil +} + +// push hands one notification to the gateways of a user's pushers. +func (a *API) push(ctx context.Context, userID string, room roomserver.PushRoom, n roomserver.Notification, outcome pushrules.Outcome) { + localpart, _, err := id.SplitUserID(userID) + if err != nil { + return + } + pushers, err := a.store.Pushers(ctx, localpart) + if err != nil { + a.log.Warn("reading pushers failed", "user_id", userID, "error", err) + return + } + if len(pushers) == 0 { + return + } + unread, err := a.store.UnreadNotifications(ctx, userID) + if err != nil { + a.log.Warn("counting unread notifications failed", "user_id", userID, "error", err) + } + + e := n.Event + note := notificationPush{ + EventID: e.EventID(), + RoomID: n.RoomID, + Type: e.Type(), + Sender: e.Sender(), + SenderDisplayName: room.Members[e.Sender()], + RoomName: room.Name, + RoomAlias: room.CanonicalAlias, + Content: e.Content(), + Counts: pushCounts{Unread: unread}, + Prio: priority(outcome), + } + if stateKey, ok := e.StateKey(); ok && e.Type() == event.TypeMember { + isTarget := stateKey == userID + note.UserIsTarget = &isTarget + if membership, err := e.Membership(); err == nil { + note.Membership = membership + } + } + for _, p := range pushers { + if p.Kind != PusherKindHTTP { + continue + } + a.gateway.enqueue(pushJob{ + localpart: localpart, + pusher: p, + note: note, + tweaks: outcome.Tweaks, + }) + } +} + +// priority is a push's prio: high for anything that makes a sound or is a +// highlight, which is what a phone should wake its user for, and low for +// the rest, which a gateway may batch to save the battery. +func priority(o pushrules.Outcome) string { + if o.Highlight { + return "high" + } + if _, ok := o.Tweaks[pushrules.TweakSound]; ok { + return "high" + } + return "low" +} + +// isLocal reports whether a user ID belongs to this server. +func (a *API) isLocal(userID string) bool { + server, err := id.ServerNameOf(userID) + return err == nil && server == a.serverName +} diff --git a/server/internal/roomserver/push.go b/server/internal/roomserver/push.go new file mode 100644 index 0000000..fd1fcf6 --- /dev/null +++ b/server/internal/roomserver/push.go @@ -0,0 +1,97 @@ +package roomserver + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/dariuszjaniak/warp/server/internal/matrix/auth" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" +) + +// PushRoom is what push rule evaluation needs to know about a room: who is +// in it under what name, what it is called, and who may notify all of it. +// +// It is read from the room's current state rather than the state at the +// event being evaluated. The push server reads it moments after the event +// lands, when the two differ only if somebody joined or renamed themselves +// in between, and reading the state at every event would cost a state +// resolution per message per room member. +type PushRoom struct { + RoomID string + + // Members maps each joined user to their display name in the room, + // empty when they have not set one. + Members map[string]string + + // Name is m.room.name, and CanonicalAlias m.room.canonical_alias's + // alias: what a push to a phone calls the room. + Name string + CanonicalAlias string + + powerLevels *auth.PowerLevels +} + +// SenderMayNotify reports whether a user has the power a notification key +// requires — "room", for an @room mention. +func (p PushRoom) SenderMayNotify(userID, key string) bool { + if p.powerLevels == nil { + return false + } + return p.powerLevels.UserLevel(userID) >= p.powerLevels.NotificationLevel(key) +} + +// PushRoomFor reads a room for the push server. +// +// It is a server-side read with no user to check against: the push server +// evaluates events on behalf of the room's members, every one of whom +// could read the same state. +func (r *API) PushRoomFor(ctx context.Context, roomID string) (PushRoom, error) { + room, rules, err := r.roomAndRules(ctx, roomID) + if err != nil { + return PushRoom{}, err + } + st, err := r.currentState(ctx, room, rules) + if err != nil { + return PushRoom{}, err + } + + out := PushRoom{RoomID: roomID, Members: map[string]string{}} + for key, e := range st { + if key.Type != event.TypeMember { + continue + } + if membership, _ := e.Membership(); membership != event.MembershipJoin { + continue + } + var content struct { + DisplayName string `json:"displayname"` + } + _ = json.Unmarshal(e.Content(), &content) + out.Members[key.StateKey] = content.DisplayName + } + + if e := st.Get("m.room.name", ""); e != nil { + var content struct { + Name string `json:"name"` + } + _ = json.Unmarshal(e.Content(), &content) + out.Name = content.Name + } + if e := st.Get("m.room.canonical_alias", ""); e != nil { + var content struct { + Alias string `json:"alias"` + } + _ = json.Unmarshal(e.Content(), &content) + out.CanonicalAlias = content.Alias + } + + if create := st.Create(); create != nil { + pl, err := auth.ResolvePowerLevels(st.PowerLevels(), create, rules) + if err != nil { + return PushRoom{}, fmt.Errorf("%w: %v", ErrRejected, err) + } + out.powerLevels = pl + } + return out, nil +} diff --git a/server/internal/setup/federation.go b/server/internal/setup/federation.go index d009e12..47cff42 100644 --- a/server/internal/setup/federation.go +++ b/server/internal/setup/federation.go @@ -13,6 +13,7 @@ import ( "github.com/dariuszjaniak/warp/server/internal/matrix/event" "github.com/dariuszjaniak/warp/server/internal/roomserver" "github.com/dariuszjaniak/warp/server/internal/storage" + "github.com/dariuszjaniak/warp/server/internal/userapi" ) // fanout is the roomserver's emit stage when more than one thing listens: @@ -62,6 +63,23 @@ func (l keyMemberships) EventPersisted(_ context.Context, n roomserver.Notificat } } +// deviceObservers fans the userapi's device changes out to every +// component that keeps state per device: the keyserver's keys and the push +// server's pushers. +type deviceObservers []userapi.DeviceObserver + +func (o deviceObservers) DevicesDeleted(ctx context.Context, localpart string, deviceIDs []string) { + for _, observer := range o { + observer.DevicesDeleted(ctx, localpart, deviceIDs) + } +} + +func (o deviceObservers) DeviceRenamed(ctx context.Context, localpart, deviceID string) { + for _, observer := range o { + observer.DeviceRenamed(ctx, localpart, deviceID) + } +} + // federationStatus adapts the module to the admin API. type federationStatus struct { module *federationapi.Module diff --git a/server/internal/setup/phase8_test.go b/server/internal/setup/phase8_test.go new file mode 100644 index 0000000..125e884 --- /dev/null +++ b/server/internal/setup/phase8_test.go @@ -0,0 +1,415 @@ +package setup_test + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "sync" + "testing" + "time" + + "github.com/dariuszjaniak/warp/server/internal/setup" +) + +// The Phase 8 tests: push rules, notification counts, pushers and the +// push gateway, through the whole stack. + +// pushSync is the part of a sync these tests read. +type pushSync struct { + NextBatch string `json:"next_batch"` + AccountData struct { + Events []struct { + Type string `json:"type"` + Content json.RawMessage `json:"content"` + } `json:"events"` + } `json:"account_data"` + Rooms struct { + Join map[string]struct { + UnreadNotifications *struct { + NotificationCount int `json:"notification_count"` + HighlightCount int `json:"highlight_count"` + } `json:"unread_notifications"` + } `json:"join"` + } `json:"rooms"` +} + +func pushSyncOnce(t *testing.T, srv *setup.Server, token, since string, timeout time.Duration) pushSync { + t.Helper() + path := "/_matrix/client/v3/sync?timeout=" + strconv.FormatInt(timeout.Milliseconds(), 10) + if since != "" { + path += "&since=" + url.QueryEscape(since) + } + rec := getWithToken(t, srv, path, token) + if rec.Code != http.StatusOK { + t.Fatalf("sync: %d %s", rec.Code, rec.Body.String()) + } + return decode[pushSync](t, rec) +} + +// waitForPush lets the push worker catch up with what has been sent. +func waitForPush(t *testing.T, srv *setup.Server) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + if err := srv.Push().WaitIdle(ctx); err != nil { + t.Fatalf("waiting for the push server: %v", err) + } +} + +func sendText(t *testing.T, srv *setup.Server, token, roomID, txn string, content map[string]any) string { + t.Helper() + resp := mustOK(t, put(t, srv, "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/send/m.room.message/"+txn, + content, token), "send") + eventID, _ := resp["event_id"].(string) + return eventID +} + +func TestPushRulesThroughTheAPI(t *testing.T) { + srv := newSigningServer(t, nil) + alice := registerThrough(t, srv, "alice") + + // The defaults, both from /pushrules/ and as account data in the + // first sync, although nothing has been stored. + all := mustOK(t, getWithToken(t, srv, "/_matrix/client/v3/pushrules/", alice), "pushrules/") + global, _ := all["global"].(map[string]any) + if override, _ := global["override"].([]any); len(override) == 0 { + t.Fatalf("pushrules/ = %v", all) + } + first := pushSyncOnce(t, srv, alice, "", 0) + var found bool + for _, e := range first.AccountData.Events { + if e.Type == "m.push_rules" { + found = true + } + } + if !found { + t.Error("the initial sync has no m.push_rules") + } + + // A new rule: created enabled, readable alone and by attribute. + mustOK(t, put(t, srv, "/_matrix/client/v3/pushrules/global/content/cake", map[string]any{ + "pattern": "cake*", "actions": []any{"notify"}, + }, alice), "put rule") + rule := mustOK(t, getWithToken(t, srv, "/_matrix/client/v3/pushrules/global/content/cake", alice), "get rule") + if rule["pattern"] != "cake*" || rule["enabled"] != true || rule["default"] != false { + t.Errorf("rule = %v", rule) + } + mustOK(t, put(t, srv, "/_matrix/client/v3/pushrules/global/content/cake/enabled", + map[string]any{"enabled": false}, alice), "disable") + if enabled := mustOK(t, getWithToken(t, srv, "/_matrix/client/v3/pushrules/global/content/cake/enabled", alice), + "get enabled"); enabled["enabled"] != false { + t.Errorf("enabled = %v", enabled) + } + + // A default rule's actions and enabled flag are the user's to change. + mustOK(t, put(t, srv, "/_matrix/client/v3/pushrules/global/override/.m.rule.master/enabled", + map[string]any{"enabled": true}, alice), "enable master") + mustOK(t, put(t, srv, "/_matrix/client/v3/pushrules/global/underride/.m.rule.message/actions", + map[string]any{"actions": []any{}}, alice), "silence messages") + if actions := mustOK(t, getWithToken(t, srv, "/_matrix/client/v3/pushrules/global/underride/.m.rule.message/actions", alice), + "get actions"); len(actions["actions"].([]any)) != 0 { + t.Errorf("actions = %v", actions) + } + + // The refusals. + for _, tc := range []struct { + name string + rec *httptest.ResponseRecorder + status int + }{ + {"a reserved rule ID", put(t, srv, "/_matrix/client/v3/pushrules/global/override/.m.rule.mine", + map[string]any{"conditions": []any{}, "actions": []any{}}, alice), http.StatusBadRequest}, + {"a content rule without a pattern", put(t, srv, "/_matrix/client/v3/pushrules/global/content/x", + map[string]any{"actions": []any{}}, alice), http.StatusBadRequest}, + {"an unknown kind", put(t, srv, "/_matrix/client/v3/pushrules/global/sideways/x", + map[string]any{"actions": []any{}}, alice), http.StatusBadRequest}, + {"another scope", getWithToken(t, srv, "/_matrix/client/v3/pushrules/device/content/cake", alice), http.StatusBadRequest}, + {"a rule that is not there", getWithToken(t, srv, "/_matrix/client/v3/pushrules/global/content/pie", alice), http.StatusNotFound}, + {"before a rule that is not there", put(t, srv, "/_matrix/client/v3/pushrules/global/content/x?before=pie", + map[string]any{"pattern": "x", "actions": []any{}}, alice), http.StatusNotFound}, + {"deleting a default rule", send(t, srv, http.MethodDelete, "/_matrix/client/v3/pushrules/global/override/.m.rule.master", + "", nil, alice, nil), http.StatusNotFound}, + {"writing the account data", put(t, srv, "/_matrix/client/v3/user/@alice:warp.example/account_data/m.push_rules", + map[string]any{"global": map[string]any{}}, alice), http.StatusMethodNotAllowed}, + } { + if tc.rec.Code != tc.status { + t.Errorf("%s: status %d, want %d: %s", tc.name, tc.rec.Code, tc.status, tc.rec.Body.String()) + } + } + + // A change wakes a blocked sync and reaches it as account data. + since := pushSyncOnce(t, srv, alice, "", 0).NextBatch + done := make(chan pushSync, 1) + go func() { done <- pushSyncOnce(t, srv, alice, since, 10*time.Second) }() + time.Sleep(100 * time.Millisecond) + start := time.Now() + if rec := send(t, srv, http.MethodDelete, "/_matrix/client/v3/pushrules/global/content/cake", "", nil, alice, nil); rec.Code != http.StatusOK { + t.Fatalf("delete: %d %s", rec.Code, rec.Body.String()) + } + woken := <-done + if time.Since(start) > 5*time.Second { + t.Error("the sync was not woken by the push rule change") + } + var rules json.RawMessage + for _, e := range woken.AccountData.Events { + if e.Type == "m.push_rules" { + rules = e.Content + } + } + var doc struct { + Global struct { + Content []struct { + RuleID string `json:"rule_id"` + } `json:"content"` + } `json:"global"` + } + _ = json.Unmarshal(rules, &doc) + if rules == nil || len(doc.Global.Content) != 1 { + t.Errorf("the woken sync's push rules = %s", rules) + } + + // The account data endpoint serves the same document. + viaAccountData := mustOK(t, getWithToken(t, srv, "/_matrix/client/v3/user/@alice:warp.example/account_data/m.push_rules", alice), + "account_data") + if _, ok := viaAccountData["global"]; !ok { + t.Errorf("account_data m.push_rules = %v", viaAccountData) + } +} + +func TestNotificationCountsThroughTheAPI(t *testing.T) { + srv := newSigningServer(t, nil) + alice := registerThrough(t, srv, "alice") + bob := registerThrough(t, srv, "bob") + + roomID := createPublicRoom(t, srv, alice) + mustOK(t, post(t, srv, "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/join", map[string]any{}, bob), "join") + waitForPush(t, srv) + since := pushSyncOnce(t, srv, alice, "", 0).NextBatch + + // Two plain messages and one that mentions Alice. + sendText(t, srv, bob, roomID, "t1", map[string]any{"msgtype": "m.text", "body": "hello"}) + sendText(t, srv, bob, roomID, "t2", map[string]any{"msgtype": "m.text", "body": "anyone?"}) + mentionID := sendText(t, srv, bob, roomID, "t3", map[string]any{"msgtype": "m.text", "body": "hey", + "m.mentions": map[string]any{"user_ids": []string{"@alice:warp.example"}}}) + // Bob's own messages do not notify Bob. + waitForPush(t, srv) + + got := pushSyncOnce(t, srv, alice, since, 0) + counts := got.Rooms.Join[roomID].UnreadNotifications + if counts == nil || counts.NotificationCount != 3 || counts.HighlightCount != 1 { + t.Fatalf("alice's counts = %+v", counts) + } + bobCounts := pushSyncOnce(t, srv, bob, "", 0).Rooms.Join[roomID].UnreadNotifications + if bobCounts == nil || bobCounts.NotificationCount != 0 { + t.Errorf("bob's counts = %+v", bobCounts) + } + + // The notification list has them, newest first. + list := mustOK(t, getWithToken(t, srv, "/_matrix/client/v3/notifications", alice), "notifications") + notes, _ := list["notifications"].([]any) + if len(notes) != 3 { + t.Fatalf("notifications = %v", list) + } + newest, _ := notes[0].(map[string]any) + if ev, _ := newest["event"].(map[string]any); ev["event_id"] != mentionID || newest["read"] != false || newest["room_id"] != roomID { + t.Errorf("newest notification = %v", newest) + } + highlights := mustOK(t, getWithToken(t, srv, "/_matrix/client/v3/notifications?only=highlight", alice), "highlights") + if notes, _ := highlights["notifications"].([]any); len(notes) != 1 { + t.Errorf("highlights = %v", highlights) + } + page := mustOK(t, getWithToken(t, srv, "/_matrix/client/v3/notifications?limit=2", alice), "page") + next, _ := page["next_token"].(string) + rest := mustOK(t, getWithToken(t, srv, "/_matrix/client/v3/notifications?limit=2&from="+next, alice), "rest") + if notes, _ := rest["notifications"].([]any); next == "" || len(notes) != 1 { + t.Errorf("paging: next %q, then %v", next, rest) + } + + // A private receipt on the newest event clears the room, and the room + // comes down the next sync with zero although nothing else happened + // in it. + since = got.NextBatch + mustOK(t, post(t, srv, "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/receipt/m.read.private/"+url.PathEscape(mentionID), + map[string]any{}, alice), "receipt") + cleared := pushSyncOnce(t, srv, alice, since, 0).Rooms.Join[roomID].UnreadNotifications + if cleared == nil || cleared.NotificationCount != 0 || cleared.HighlightCount != 0 { + t.Errorf("after the receipt: %+v", cleared) + } + if notes, _ := mustOK(t, getWithToken(t, srv, "/_matrix/client/v3/notifications", alice), "notifications")["notifications"].([]any); len(notes) != 3 || notes[0].(map[string]any)["read"] != true { + t.Errorf("after the receipt the list is %v", notes) + } + + // Silencing the message rules stops plain messages counting. In a + // room of two that is the one-to-one rule as well, which comes first. + for _, rule := range []string{".m.rule.room_one_to_one", ".m.rule.message"} { + mustOK(t, put(t, srv, "/_matrix/client/v3/pushrules/global/underride/"+rule+"/enabled", + map[string]any{"enabled": false}, alice), "disable "+rule) + } + sendText(t, srv, bob, roomID, "t4", map[string]any{"msgtype": "m.text", "body": "quiet now"}) + waitForPush(t, srv) + if c := pushSyncOnce(t, srv, alice, "", 0).Rooms.Join[roomID].UnreadNotifications; c == nil || c.NotificationCount != 0 { + t.Errorf("with the message rule off: %+v", c) + } +} + +// fakeGateway records the pushes it is sent, and rejects push keys on +// request. +type fakeGateway struct { + mu sync.Mutex + pushes []map[string]any + reject string + got chan struct{} +} + +func newFakeGateway(t *testing.T) (*fakeGateway, string) { + g := &fakeGateway{got: make(chan struct{}, 16)} + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/_matrix/push/v1/notify" { + http.NotFound(w, r) + return + } + body, _ := io.ReadAll(r.Body) + var push map[string]any + _ = json.Unmarshal(body, &push) + g.mu.Lock() + g.pushes = append(g.pushes, push) + reject := g.reject + g.mu.Unlock() + rejected := []string{} + if reject != "" { + rejected = append(rejected, reject) + } + _ = json.NewEncoder(w).Encode(map[string]any{"rejected": rejected}) + g.got <- struct{}{} + })) + t.Cleanup(ts.Close) + return g, ts.URL + "/_matrix/push/v1/notify" +} + +func (g *fakeGateway) wait(t *testing.T) map[string]any { + t.Helper() + select { + case <-g.got: + case <-time.After(10 * time.Second): + t.Fatal("the gateway was not pushed to") + } + g.mu.Lock() + defer g.mu.Unlock() + return g.pushes[len(g.pushes)-1] +} + +func TestPushersThroughTheAPI(t *testing.T) { + srv := newSigningServer(t, nil) + alice := registerThrough(t, srv, "alice") + bob := registerThrough(t, srv, "bob") + gateway, gatewayURL := newFakeGateway(t) + + pusher := func(key string, data map[string]any) map[string]any { + return map[string]any{ + "pushkey": key, "kind": "http", "app_id": "org.example.app", + "app_display_name": "Example", "device_display_name": "Phone", "lang": "en", + "data": data, + } + } + + // The refusals first. + for _, tc := range []struct { + name string + body map[string]any + }{ + {"a gateway URL with the wrong path", pusher("k", map[string]any{"url": "https://push.example/notify"})}, + {"no gateway URL", pusher("k", map[string]any{})}, + {"an email pusher", func() map[string]any { p := pusher("k", map[string]any{}); p["kind"] = "email"; return p }()}, + {"no kind", func() map[string]any { + p := pusher("k", map[string]any{"url": gatewayURL}) + delete(p, "kind") + return p + }()}, + } { + if rec := post(t, srv, "/_matrix/client/v3/pushers/set", tc.body, alice); rec.Code != http.StatusBadRequest { + t.Errorf("%s: status %d: %s", tc.name, rec.Code, rec.Body.String()) + } + } + + mustOK(t, post(t, srv, "/_matrix/client/v3/pushers/set", + pusher("key1", map[string]any{"url": gatewayURL, "custom": "x"}), alice), "pushers/set") + listed := mustOK(t, getWithToken(t, srv, "/_matrix/client/v3/pushers", alice), "pushers") + if pushers, _ := listed["pushers"].([]any); len(pushers) != 1 || pushers[0].(map[string]any)["pushkey"] != "key1" { + t.Fatalf("pushers = %v", listed) + } + + roomID := createPublicRoom(t, srv, alice) + mustOK(t, post(t, srv, "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/join", map[string]any{}, bob), "join") + mustOK(t, put(t, srv, "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/state/m.room.name/", + map[string]any{"name": "Tea"}, alice), "name") + waitForPush(t, srv) + + eventID := sendText(t, srv, bob, roomID, "p1", map[string]any{"msgtype": "m.text", "body": "hello"}) + push := gateway.wait(t) + note, _ := push["notification"].(map[string]any) + devices, _ := note["devices"].([]any) + device, _ := devices[0].(map[string]any) + data, _ := device["data"].(map[string]any) + counts, _ := note["counts"].(map[string]any) + if note["event_id"] != eventID || note["room_id"] != roomID || note["sender"] != "@bob:warp.example" || + note["room_name"] != "Tea" || note["type"] != "m.room.message" || counts["unread"] != float64(1) { + t.Errorf("notification = %v", note) + } + if device["pushkey"] != "key1" || device["app_id"] != "org.example.app" || data["custom"] != "x" || data["url"] != nil { + t.Errorf("device = %v", device) + } + if content, _ := note["content"].(map[string]any); content["body"] != "hello" { + t.Errorf("content = %v", note["content"]) + } + + // event_id_only sends the IDs and nothing of the event. + mustOK(t, post(t, srv, "/_matrix/client/v3/pushers/set", + pusher("key1", map[string]any{"url": gatewayURL, "format": "event_id_only"}), alice), "pushers/set") + sendText(t, srv, bob, roomID, "p2", map[string]any{"msgtype": "m.text", "body": "secret"}) + note, _ = gateway.wait(t)["notification"].(map[string]any) + if note["content"] != nil || note["sender"] != nil || note["event_id"] == nil { + t.Errorf("event_id_only notification = %v", note) + } + + // A push key the gateway rejects is removed. + gateway.mu.Lock() + gateway.reject = "key1" + gateway.mu.Unlock() + sendText(t, srv, bob, roomID, "p3", map[string]any{"msgtype": "m.text", "body": "again"}) + gateway.wait(t) + deadline := time.Now().Add(5 * time.Second) + for { + pushers, _ := mustOK(t, getWithToken(t, srv, "/_matrix/client/v3/pushers", alice), "pushers")["pushers"].([]any) + if len(pushers) == 0 { + break + } + if time.Now().After(deadline) { + t.Fatalf("the rejected pusher is still there: %v", pushers) + } + time.Sleep(20 * time.Millisecond) + } + + // Logging out takes the session's pushers with it, and kind null + // removes one. + mustOK(t, post(t, srv, "/_matrix/client/v3/pushers/set", + pusher("key2", map[string]any{"url": gatewayURL}), alice), "pushers/set") + removal := pusher("key2", nil) + removal["kind"] = nil + mustOK(t, post(t, srv, "/_matrix/client/v3/pushers/set", removal, alice), "remove pusher") + if pushers, _ := mustOK(t, getWithToken(t, srv, "/_matrix/client/v3/pushers", alice), "pushers")["pushers"].([]any); len(pushers) != 0 { + t.Errorf("after kind null: %v", pushers) + } + mustOK(t, post(t, srv, "/_matrix/client/v3/pushers/set", + pusher("key3", map[string]any{"url": gatewayURL}), alice), "pushers/set") + if pushers, _ := srv.Push().Pushers(t.Context(), "alice"); len(pushers) != 1 { + t.Fatalf("before logout: %v", pushers) + } + mustOK(t, post(t, srv, "/_matrix/client/v3/logout", map[string]any{}, alice), "logout") + if pushers, _ := srv.Push().Pushers(t.Context(), "alice"); len(pushers) != 0 { + t.Errorf("after logout: %v", pushers) + } +} diff --git a/server/internal/setup/routes.go b/server/internal/setup/routes.go index ae69d1b..4a11bbb 100644 --- a/server/internal/setup/routes.go +++ b/server/internal/setup/routes.go @@ -12,13 +12,11 @@ import ( // // Keep it honest: a version belongs here once the endpoints a client // actually uses from it work. Accounts, sessions, devices, profiles, -// account data, rooms, the event stream, media (Phase 6) and end-to-end -// encryption (Phase 7) all do, which is enough for a client to be used. -// Advertising v1.11 or later is also what makes a client use the -// authenticated media endpoints, the only download endpoints Warp serves -// by default. What is still missing is push (Phase 8); a client discovers -// that from the endpoint it needs rather than from this list, which is why -// its absence does not make a version listed here a lie. +// account data, rooms, the event stream, media (Phase 6), end-to-end +// encryption (Phase 7) and push (Phase 8) all do, which is enough for a +// client to be used. Advertising v1.11 or later is also what makes a client +// use the authenticated media endpoints, the only download endpoints Warp +// serves by default. var SpecVersions = []string{"v1.13", "v1.14"} // UnstableFeatures advertises MSC support. Simplified Sliding Sync diff --git a/server/internal/setup/server.go b/server/internal/setup/server.go index 2d9a29f..5763223 100644 --- a/server/internal/setup/server.go +++ b/server/internal/setup/server.go @@ -25,6 +25,7 @@ import ( "github.com/dariuszjaniak/warp/server/internal/matrix/signing" "github.com/dariuszjaniak/warp/server/internal/mediaapi" "github.com/dariuszjaniak/warp/server/internal/metrics" + "github.com/dariuszjaniak/warp/server/internal/pushserver" "github.com/dariuszjaniak/warp/server/internal/roomserver" "github.com/dariuszjaniak/warp/server/internal/settings" "github.com/dariuszjaniak/warp/server/internal/storage" @@ -48,6 +49,7 @@ type Server struct { rooms *roomserver.API sync *syncapi.API keys *keyserver.API + push *pushserver.API media *mediaapi.API clientAPI *clientapi.ClientAPI adminAPI *adminapi.AdminAPI @@ -294,14 +296,39 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) _ = s.closeDB() return nil, err } - s.users.SetDeviceObserver(s.keys) emit.add(keyMemberships{keys: s.keys}) + // The push server is the last listener on the emit stage. It + // evaluates every event against the push rules of every local + // member of its room, on a worker of its own, and it is what the + // sync reads the unread counts from. Logging a device out takes + // its pushers with it, which makes the userapi's device observer + // two of them. + s.push, err = pushserver.New(pushserver.Options{ + ServerName: cfg.Global.ServerName, + Store: s.db.Push(), + Accounts: s.users, + Rooms: s.rooms, + Notifier: notifier, + PushEnabled: cfg.Push.Enabled, + IncludeContent: cfg.Push.IncludeContent, + GatewayTimeout: cfg.Push.GatewayTimeout, + NotificationRetention: cfg.Push.NotificationRetention, + Logger: log, + }) + if err != nil { + _ = s.closeDB() + return nil, err + } + emit.add(s.push) + s.users.SetDeviceObserver(deviceObservers{s.keys, s.push}) + s.sync, err = syncapi.New(syncapi.Options{ ServerName: cfg.Global.ServerName, Rooms: s.rooms, Accounts: s.users, Keys: s.keys, + Push: s.push, Store: s.db.Sync(), Notifier: notifier, PresenceEnabled: cfg.Sync.Presence, @@ -353,6 +380,7 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) Sync: s.sync, Keys: s.keys, Media: s.media, + Push: s.push, LegacyMediaDownloads: cfg.Media.LegacyUnauthenticatedDownloads, Settings: settingsService, Federation: s.federation, @@ -445,6 +473,10 @@ func (s *Server) Sync() *syncapi.API { return s.sync } // signing key. func (s *Server) Keys() *keyserver.API { return s.keys } +// Push exposes the push server, or nil when the server started without a +// signing key. +func (s *Server) Push() *pushserver.API { return s.push } + // Media exposes the media repository. func (s *Server) Media() *mediaapi.API { return s.media } @@ -587,6 +619,11 @@ func (s *Server) Close() error { // shutting down should not wait for them. s.sync.Close() } + if s.push != nil { + // The push worker writes notifications, and the database is about + // to close. + s.push.Close() + } return s.closeDB() } diff --git a/server/internal/storage/pocketbase/pocketbase.go b/server/internal/storage/pocketbase/pocketbase.go index 4f3b2d5..ee6412c 100644 --- a/server/internal/storage/pocketbase/pocketbase.go +++ b/server/internal/storage/pocketbase/pocketbase.go @@ -69,6 +69,9 @@ func (*Database) Media() storage.MediaStore { return notImplementedMedia{} } // Keys implements storage.Database. func (*Database) Keys() storage.KeyStore { return notImplementedKeys{} } +// Push implements storage.Database. +func (*Database) Push() storage.PushStore { return notImplementedPush{} } + // Close implements storage.Database. func (*Database) Close() error { return nil } @@ -605,3 +608,50 @@ func (notImplementedKeys) RemoteKeyUsers(context.Context) ([]string, error) { func (notImplementedKeys) DeleteRemoteKeys(context.Context, []string) error { return storage.ErrNotImplemented } + +// notImplementedPush is the push store until the driver has one. +type notImplementedPush struct{} + +func (notImplementedPush) Pushers(context.Context, string) ([]storage.Pusher, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedPush) SetPusher(context.Context, storage.Pusher, bool) error { + return storage.ErrNotImplemented +} + +func (notImplementedPush) DeletePusher(context.Context, string, string, string) error { + return storage.ErrNotImplemented +} + +func (notImplementedPush) DeletePushersForDevices(context.Context, string, []string) error { + return storage.ErrNotImplemented +} + +func (notImplementedPush) AddNotifications(context.Context, []storage.NewNotification) (int64, error) { + return 0, storage.ErrNotImplemented +} + +func (notImplementedPush) MarkNotificationsRead(context.Context, string, string, int64) (int64, bool, error) { + return 0, false, storage.ErrNotImplemented +} + +func (notImplementedPush) NotificationCounts(context.Context, string, []string) (map[string]storage.NotificationCounts, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedPush) NotificationCountChanges(context.Context, string, int64, int64) ([]string, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedPush) UnreadNotifications(context.Context, string) (int, error) { + return 0, storage.ErrNotImplemented +} + +func (notImplementedPush) Notifications(context.Context, string, int64, int, bool) ([]storage.Notification, error) { + return nil, storage.ErrNotImplemented +} + +func (notImplementedPush) PruneNotifications(context.Context, int64) (int64, error) { + return 0, storage.ErrNotImplemented +} diff --git a/server/internal/storage/postgres/migrations/0010_push.sql b/server/internal/storage/postgres/migrations/0010_push.sql new file mode 100644 index 0000000..720c970 --- /dev/null +++ b/server/internal/storage/postgres/migrations/0010_push.sql @@ -0,0 +1,70 @@ +-- Push: pushers, and the notifications push rules produce. +-- +-- Push rules themselves are not here. They are the m.push_rules account +-- data, which is where the spec has clients read them and where /sync +-- already carries account data from; a table of their own would be a +-- second copy to keep in step with that one. + +-- The notification count stream: one counter, next to its tables, read +-- with the other streams so that a sync token names one moment in all of +-- them. +CREATE TABLE IF NOT EXISTS push_sequences ( + name TEXT NOT NULL PRIMARY KEY, + value BIGINT NOT NULL +); + +INSERT INTO push_sequences (name, value) VALUES ('notification', 0) ON CONFLICT (name) DO NOTHING; + +-- Pushers. A pusher is one app on one device, named by its app ID and push +-- key; device_id is the session that registered it, so that logging the +-- session out removes it. +CREATE TABLE IF NOT EXISTS push_pushers ( + localpart TEXT NOT NULL, + app_id TEXT NOT NULL, + pushkey TEXT NOT NULL, + device_id TEXT NOT NULL DEFAULT '', + kind TEXT NOT NULL, + app_display_name TEXT NOT NULL DEFAULT '', + device_display_name TEXT NOT NULL DEFAULT '', + profile_tag TEXT NOT NULL DEFAULT '', + lang TEXT NOT NULL DEFAULT '', + data_json TEXT NOT NULL, + pushkey_ts_ms BIGINT NOT NULL, + PRIMARY KEY (localpart, app_id, pushkey) +); + +CREATE INDEX IF NOT EXISTS push_pushers_key_idx ON push_pushers (app_id, pushkey); + +-- One row per event that notified a user. id is the position the row was +-- written at in the notification stream, which also orders a user's +-- notifications for GET /notifications. event_pos is the event's position +-- in the room event stream, which a read receipt is compared against. +CREATE TABLE IF NOT EXISTS push_notifications ( + id BIGINT NOT NULL PRIMARY KEY, + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + event_pos BIGINT NOT NULL, + ts_ms BIGINT NOT NULL, + highlight BOOLEAN NOT NULL DEFAULT FALSE, + actions TEXT NOT NULL, + is_read BOOLEAN NOT NULL DEFAULT FALSE +); + +CREATE INDEX IF NOT EXISTS push_notifications_user_idx ON push_notifications (user_id, id); +CREATE INDEX IF NOT EXISTS push_notifications_unread_idx ON push_notifications (user_id, room_id, is_read, event_pos); +CREATE INDEX IF NOT EXISTS push_notifications_prune_idx ON push_notifications (is_read, ts_ms); + +-- A user's unread counts in a room, kept current by the writes above +-- rather than counted on every sync, and stamped with the notification +-- stream position of the write that last moved them. +CREATE TABLE IF NOT EXISTS push_room_counts ( + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + notification_count BIGINT NOT NULL, + highlight_count BIGINT NOT NULL, + stream_pos BIGINT NOT NULL, + PRIMARY KEY (user_id, room_id) +); + +CREATE INDEX IF NOT EXISTS push_room_counts_stream_idx ON push_room_counts (user_id, stream_pos); diff --git a/server/internal/storage/push.go b/server/internal/storage/push.go new file mode 100644 index 0000000..e478b7b --- /dev/null +++ b/server/internal/storage/push.go @@ -0,0 +1,130 @@ +package storage + +import ( + "context" + "errors" +) + +// ErrNoSuchPusher is a pusher that does not exist. +var ErrNoSuchPusher = errors.New("no such pusher") + +// Pusher is one place a user's notifications are pushed to: an app on a +// device, reached through its push gateway. +type Pusher struct { + Localpart string + + // DeviceID is the session that registered the pusher, so that logging + // the device out removes it. Empty for a pusher whose session is not + // known. + DeviceID string + + Kind string + AppID string + PushKey string + + AppDisplayName string + DeviceDisplayName string + ProfileTag string + Lang string + + // Data is the pusher's data object as the client sent it: the gateway + // URL, the payload format, and anything else the app wants passed to + // its gateway. + Data []byte + + // PushKeyTSMS is when the push key was last set, which a gateway uses + // to tell an old registration of a key from a newer one. + PushKeyTSMS int64 +} + +// NewNotification is one event that notifies one user. +type NewNotification struct { + UserID string + RoomID string + EventID string + + // EventPos is the event's position in the room event stream, which is + // what a read receipt is compared against. + EventPos int64 + + TimestampMS int64 + Highlight bool + + // Actions are the matched rule's actions, as JSON, which GET + // /notifications reports. + Actions []byte +} + +// Notification is a stored notification. +type Notification struct { + NewNotification + + // ID orders a user's notifications, newest highest, and is what GET + // /notifications pages by. + ID int64 + + Read bool +} + +// NotificationCounts is a room's unread notifications for one user. +type NotificationCounts struct { + Notifications int + Highlights int +} + +// PushStore persists pushers and the notifications push rules produce. +// +// Notification counts are a stream of their own: every write that changes +// a user's counts in a room stamps the room with a new position, inside +// the transaction that changes them, so that an incremental sync can ask +// which rooms' counts moved since its token. A count moves without an +// event — a read receipt sent from another device sets it back to zero — +// and a sync that only looked at events would go on showing the old one. +type PushStore interface { + // Pushers lists a user's pushers. + Pushers(ctx context.Context, localpart string) ([]Pusher, error) + + // SetPusher creates or replaces the user's pusher with the same app ID + // and push key. Unless appendOthers is set, any other user's pusher + // with the same app ID and push key is removed: a push key names a + // device, and the device is now this user's. + SetPusher(ctx context.Context, p Pusher, appendOthers bool) error + + // DeletePusher removes one of a user's pushers. Removing one that is + // not there is not an error. + DeletePusher(ctx context.Context, localpart, appID, pushKey string) error + + // DeletePushersForDevices removes the pushers registered by the given + // sessions, or by every session of the user when deviceIDs is empty. + DeletePushersForDevices(ctx context.Context, localpart string, deviceIDs []string) error + + // AddNotifications records notifications and moves the counts of + // each (user, room) they are for, returning the position the last + // count moved to. + AddNotifications(ctx context.Context, ns []NewNotification) (int64, error) + + // MarkNotificationsRead marks a user's notifications in a room up to + // and including an event position as read, and reports whether the + // counts moved. + MarkNotificationsRead(ctx context.Context, userID, roomID string, upTo int64) (pos int64, moved bool, err error) + + // NotificationCounts reads a user's unread counts in the given rooms. + // A room with nothing unread is in the result with zero counts. + NotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]NotificationCounts, error) + + // NotificationCountChanges lists the rooms whose counts for a user + // moved after from and at or before to. + NotificationCountChanges(ctx context.Context, userID string, from, to int64) ([]string, error) + + // UnreadNotifications is a user's unread notifications across every + // room, which a push tells the device to show as its badge. + UnreadNotifications(ctx context.Context, userID string) (int, error) + + // Notifications pages through a user's notifications, newest first, + // starting below the ID before (0 for the newest). + Notifications(ctx context.Context, userID string, before int64, limit int, onlyHighlight bool) ([]Notification, error) + + // PruneNotifications removes read notifications older than a time, + // returning how many went. + PruneNotifications(ctx context.Context, beforeMS int64) (int64, error) +} diff --git a/server/internal/storage/push_test.go b/server/internal/storage/push_test.go new file mode 100644 index 0000000..2d247f9 --- /dev/null +++ b/server/internal/storage/push_test.go @@ -0,0 +1,218 @@ +package storage_test + +import ( + "database/sql" + "errors" + "os" + "slices" + "testing" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// migratedPushDB opens a migrated database with empty push tables. +func migratedPushDB(t *testing.T, driver string) storage.Database { + t.Helper() + db := openTestDB(t, driver) + if err := db.Migrate(t.Context()); err != nil { + t.Fatalf("Migrate: %v", err) + } + if driver == storage.DriverPostgres { + pool, err := sql.Open("pgx", os.Getenv("WARP_TEST_POSTGRES_DSN")) + if err != nil { + t.Fatalf("open postgres to reset the push tables: %v", err) + } + defer func() { _ = pool.Close() }() + if _, err := pool.ExecContext(t.Context(), + `TRUNCATE push_pushers, push_notifications, push_room_counts`); err != nil { + t.Fatalf("reset the push tables: %v", err) + } + if _, err := pool.ExecContext(t.Context(), + `UPDATE push_sequences SET value = 0`); err != nil { + t.Fatalf("reset the push counter: %v", err) + } + } + return db +} + +func TestPushers(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + push := migratedPushDB(t, driver).Push() + + pusher := func(localpart, device, key string) storage.Pusher { + return storage.Pusher{ + Localpart: localpart, DeviceID: device, Kind: "http", AppID: "app", PushKey: key, + AppDisplayName: "App", DeviceDisplayName: "Phone", Lang: "en", + Data: []byte(`{"url":"https://push.example/_matrix/push/v1/notify"}`), PushKeyTSMS: 1, + } + } + if err := push.SetPusher(ctx, pusher("alice", "D1", "k1"), false); err != nil { + t.Fatal(err) + } + if err := push.SetPusher(ctx, pusher("alice", "D2", "k2"), false); err != nil { + t.Fatal(err) + } + + // Replacing keeps one row and takes the new values. + updated := pusher("alice", "D1", "k1") + updated.Lang = "fr" + if err := push.SetPusher(ctx, updated, false); err != nil { + t.Fatal(err) + } + got, err := push.Pushers(ctx, "alice") + if err != nil || len(got) != 2 || got[0].Lang != "fr" || string(got[0].Data) != string(updated.Data) { + t.Fatalf("Pushers = %+v, %v", got, err) + } + + // The same key for another user takes it over, unless appending. + if err := push.SetPusher(ctx, pusher("bob", "B1", "k1"), true); err != nil { + t.Fatal(err) + } + if got, _ := push.Pushers(ctx, "alice"); len(got) != 2 { + t.Errorf("appending removed alice's pusher: %+v", got) + } + if err := push.SetPusher(ctx, pusher("bob", "B1", "k1"), false); err != nil { + t.Fatal(err) + } + if got, _ := push.Pushers(ctx, "alice"); len(got) != 1 || got[0].PushKey != "k2" { + t.Errorf("taking the key over left alice with %+v", got) + } + + if err := push.DeletePushersForDevices(ctx, "alice", []string{"D2"}); err != nil { + t.Fatal(err) + } + if got, _ := push.Pushers(ctx, "alice"); len(got) != 0 { + t.Errorf("after deleting the device: %+v", got) + } + if err := push.DeletePusher(ctx, "bob", "app", "k1"); err != nil { + t.Fatal(err) + } + if err := push.DeletePusher(ctx, "bob", "app", "k1"); err != nil { + t.Errorf("deleting a pusher twice: %v", err) + } + if got, _ := push.Pushers(ctx, "bob"); len(got) != 0 { + t.Errorf("bob's pushers after the delete: %+v", got) + } + }) + } +} + +func TestNotifications(t *testing.T) { + const alice, bob = "@alice:warp.example", "@bob:warp.example" + const room1, room2 = "!one:warp.example", "!two:warp.example" + + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + db := migratedPushDB(t, driver) + push := db.Push() + + start, err := db.Sync().StreamPositions(ctx) + if err != nil { + t.Fatal(err) + } + + note := func(user, room, event string, pos int64, highlight bool) storage.NewNotification { + return storage.NewNotification{UserID: user, RoomID: room, EventID: event, EventPos: pos, + TimestampMS: 1000 + pos, Highlight: highlight, Actions: []byte(`["notify"]`)} + } + last, err := push.AddNotifications(ctx, []storage.NewNotification{ + note(alice, room1, "$a", 10, false), + note(alice, room1, "$b", 11, true), + note(alice, room2, "$c", 12, false), + note(bob, room1, "$b", 11, false), + }) + if err != nil || last != start.Notification+4 { + t.Fatalf("AddNotifications = %d, %v", last, err) + } + if positions, _ := db.Sync().StreamPositions(ctx); positions.Notification != last { + t.Errorf("the stream is at %d, want %d", positions.Notification, last) + } + + counts, err := push.NotificationCounts(ctx, alice, []string{room1, room2, "!none:x"}) + if err != nil { + t.Fatal(err) + } + if counts[room1] != (storage.NotificationCounts{Notifications: 2, Highlights: 1}) || + counts[room2] != (storage.NotificationCounts{Notifications: 1}) || + counts["!none:x"] != (storage.NotificationCounts{}) { + t.Errorf("counts = %+v", counts) + } + if total, _ := push.UnreadNotifications(ctx, alice); total != 3 { + t.Errorf("UnreadNotifications = %d", total) + } + + changed, err := push.NotificationCountChanges(ctx, alice, start.Notification, last) + if err != nil || !slices.Equal(changed, []string{room1, room2}) { + t.Errorf("NotificationCountChanges = %v, %v", changed, err) + } + + // Reading up to the first event clears it alone, and moves the + // stream for that room only. + pos, moved, err := push.MarkNotificationsRead(ctx, alice, room1, 10) + if err != nil || !moved || pos != last+1 { + t.Fatalf("MarkNotificationsRead = %d, %v, %v", pos, moved, err) + } + counts, _ = push.NotificationCounts(ctx, alice, []string{room1}) + if counts[room1] != (storage.NotificationCounts{Notifications: 1, Highlights: 1}) { + t.Errorf("after reading one: %+v", counts) + } + if changed, _ := push.NotificationCountChanges(ctx, alice, last, pos); !slices.Equal(changed, []string{room1}) { + t.Errorf("changes after the read = %v", changed) + } + + // Reading the same again moves nothing. + if again, moved, err := push.MarkNotificationsRead(ctx, alice, room1, 10); err != nil || moved || again != pos { + t.Errorf("reading again = %d, %v, %v", again, moved, err) + } + + if _, _, err := push.MarkNotificationsRead(ctx, alice, room1, 100); err != nil { + t.Fatal(err) + } + counts, _ = push.NotificationCounts(ctx, alice, []string{room1}) + if counts[room1] != (storage.NotificationCounts{}) { + t.Errorf("after reading everything: %+v", counts) + } + // Bob's are his own. + if counts, _ := push.NotificationCounts(ctx, bob, []string{room1}); counts[room1].Notifications != 1 { + t.Errorf("bob's counts = %+v", counts) + } + + all, err := push.Notifications(ctx, alice, 0, 10, false) + if err != nil || len(all) != 3 || all[0].EventID != "$c" || all[0].Read || !all[2].Read || + string(all[1].Actions) != `["notify"]` || !all[1].Highlight { + t.Fatalf("Notifications = %+v, %v", all, err) + } + page, _ := push.Notifications(ctx, alice, all[0].ID, 1, false) + if len(page) != 1 || page[0].EventID != "$b" { + t.Errorf("the second page = %+v", page) + } + highlights, _ := push.Notifications(ctx, alice, 0, 10, true) + if len(highlights) != 1 || highlights[0].EventID != "$b" { + t.Errorf("highlights = %+v", highlights) + } + + // Pruning takes only read notifications older than the cutoff. + n, err := push.PruneNotifications(ctx, 1011) + if err != nil || n != 1 { + t.Errorf("PruneNotifications = %d, %v", n, err) + } + if rest, _ := push.Notifications(ctx, alice, 0, 10, false); len(rest) != 2 { + t.Errorf("after pruning: %+v", rest) + } + }) + } +} + +func TestPushNotImplementedOnPocketBase(t *testing.T) { + db, err := storage.Open(t.Context(), storage.Config{Driver: storage.DriverPocketBase}, nil) + if err != nil { + t.Skipf("pocketbase: %v", err) + } + defer func() { _ = db.Close() }() + if _, err := db.Push().Pushers(t.Context(), "alice"); !errors.Is(err, storage.ErrNotImplemented) { + t.Errorf("Pushers = %v", err) + } +} diff --git a/server/internal/storage/sqlite/migrations/0010_push.sql b/server/internal/storage/sqlite/migrations/0010_push.sql new file mode 100644 index 0000000..c6c49e4 --- /dev/null +++ b/server/internal/storage/sqlite/migrations/0010_push.sql @@ -0,0 +1,70 @@ +-- Push: pushers, and the notifications push rules produce. +-- +-- Push rules themselves are not here. They are the m.push_rules account +-- data, which is where the spec has clients read them and where /sync +-- already carries account data from; a table of their own would be a +-- second copy to keep in step with that one. + +-- The notification count stream: one counter, next to its tables, read +-- with the other streams so that a sync token names one moment in all of +-- them. +CREATE TABLE IF NOT EXISTS push_sequences ( + name TEXT NOT NULL PRIMARY KEY, + value BIGINT NOT NULL +); + +INSERT INTO push_sequences (name, value) VALUES ('notification', 0) ON CONFLICT (name) DO NOTHING; + +-- Pushers. A pusher is one app on one device, named by its app ID and push +-- key; device_id is the session that registered it, so that logging the +-- session out removes it. +CREATE TABLE IF NOT EXISTS push_pushers ( + localpart TEXT NOT NULL, + app_id TEXT NOT NULL, + pushkey TEXT NOT NULL, + device_id TEXT NOT NULL DEFAULT '', + kind TEXT NOT NULL, + app_display_name TEXT NOT NULL DEFAULT '', + device_display_name TEXT NOT NULL DEFAULT '', + profile_tag TEXT NOT NULL DEFAULT '', + lang TEXT NOT NULL DEFAULT '', + data_json TEXT NOT NULL, + pushkey_ts_ms BIGINT NOT NULL, + PRIMARY KEY (localpart, app_id, pushkey) +); + +CREATE INDEX IF NOT EXISTS push_pushers_key_idx ON push_pushers (app_id, pushkey); + +-- One row per event that notified a user. id is the position the row was +-- written at in the notification stream, which also orders a user's +-- notifications for GET /notifications. event_pos is the event's position +-- in the room event stream, which a read receipt is compared against. +CREATE TABLE IF NOT EXISTS push_notifications ( + id BIGINT NOT NULL PRIMARY KEY, + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + event_pos BIGINT NOT NULL, + ts_ms BIGINT NOT NULL, + highlight BOOLEAN NOT NULL DEFAULT 0, + actions TEXT NOT NULL, + is_read BOOLEAN NOT NULL DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS push_notifications_user_idx ON push_notifications (user_id, id); +CREATE INDEX IF NOT EXISTS push_notifications_unread_idx ON push_notifications (user_id, room_id, is_read, event_pos); +CREATE INDEX IF NOT EXISTS push_notifications_prune_idx ON push_notifications (is_read, ts_ms); + +-- A user's unread counts in a room, kept current by the writes above +-- rather than counted on every sync, and stamped with the notification +-- stream position of the write that last moved them. +CREATE TABLE IF NOT EXISTS push_room_counts ( + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + notification_count BIGINT NOT NULL, + highlight_count BIGINT NOT NULL, + stream_pos BIGINT NOT NULL, + PRIMARY KEY (user_id, room_id) +); + +CREATE INDEX IF NOT EXISTS push_room_counts_stream_idx ON push_room_counts (user_id, stream_pos); diff --git a/server/internal/storage/sqlstore/push.go b/server/internal/storage/sqlstore/push.go new file mode 100644 index 0000000..743ba4a --- /dev/null +++ b/server/internal/storage/sqlstore/push.go @@ -0,0 +1,372 @@ +package sqlstore + +import ( + "context" + "database/sql" + "fmt" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// The push store's sequence table and its one counter. +const ( + pushSequences = "push_sequences" + seqNotification = "notification" +) + +// Push is the shared implementation of storage.PushStore. +type Push struct { + db *sql.DB + rebind func(string) string +} + +// NewPush builds a push store. +func NewPush(db *sql.DB, rebind func(string) string) *Push { + if rebind == nil { + rebind = func(query string) string { return query } + } + return &Push{db: db, rebind: rebind} +} + +func (s *Push) q(query string) string { return s.rebind(query) } + +// tx runs fn in a transaction, rolling back on any error. +func (s *Push) tx(ctx context.Context, fn func(*sql.Tx) error) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin transaction: %w", err) + } + defer func() { _ = tx.Rollback() }() + + if err := fn(tx); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit transaction: %w", err) + } + return nil +} + +const ( + pusherColumns = `localpart, app_id, pushkey, device_id, kind, app_display_name, + device_display_name, profile_tag, lang, data_json, pushkey_ts_ms` + + selectPushersSQL = `SELECT ` + pusherColumns + ` FROM push_pushers + WHERE localpart = ? ORDER BY app_id, pushkey` + + upsertPusherSQL = `INSERT INTO push_pushers (` + pusherColumns + `) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (localpart, app_id, pushkey) DO UPDATE SET + device_id = excluded.device_id, + kind = excluded.kind, + app_display_name = excluded.app_display_name, + device_display_name = excluded.device_display_name, + profile_tag = excluded.profile_tag, + lang = excluded.lang, + data_json = excluded.data_json, + pushkey_ts_ms = excluded.pushkey_ts_ms` + + deleteOtherUsersPushersSQL = `DELETE FROM push_pushers + WHERE app_id = ? AND pushkey = ? AND localpart <> ?` + + deletePusherSQL = `DELETE FROM push_pushers + WHERE localpart = ? AND app_id = ? AND pushkey = ?` + + deleteAllPushersSQL = `DELETE FROM push_pushers WHERE localpart = ?` +) + +// Pushers implements storage.PushStore. +func (s *Push) Pushers(ctx context.Context, localpart string) ([]storage.Pusher, error) { + rows, err := s.db.QueryContext(ctx, s.q(selectPushersSQL), localpart) + if err != nil { + return nil, fmt.Errorf("read the pushers of %q: %w", localpart, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.Pusher + for rows.Next() { + var p storage.Pusher + var data string + if err := rows.Scan(&p.Localpart, &p.AppID, &p.PushKey, &p.DeviceID, &p.Kind, &p.AppDisplayName, + &p.DeviceDisplayName, &p.ProfileTag, &p.Lang, &data, &p.PushKeyTSMS); err != nil { + return nil, fmt.Errorf("read the pushers of %q: %w", localpart, err) + } + p.Data = []byte(data) + out = append(out, p) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read the pushers of %q: %w", localpart, err) + } + return out, nil +} + +// SetPusher implements storage.PushStore. +func (s *Push) SetPusher(ctx context.Context, p storage.Pusher, appendOthers bool) error { + return s.tx(ctx, func(tx *sql.Tx) error { + if !appendOthers { + if _, err := tx.ExecContext(ctx, s.q(deleteOtherUsersPushersSQL), p.AppID, p.PushKey, p.Localpart); err != nil { + return fmt.Errorf("remove other users' pushers for the key: %w", err) + } + } + if _, err := tx.ExecContext(ctx, s.q(upsertPusherSQL), p.Localpart, p.AppID, p.PushKey, p.DeviceID, p.Kind, + p.AppDisplayName, p.DeviceDisplayName, p.ProfileTag, p.Lang, string(p.Data), p.PushKeyTSMS); err != nil { + return fmt.Errorf("write the pusher of %q: %w", p.Localpart, err) + } + return nil + }) +} + +// DeletePusher implements storage.PushStore. +func (s *Push) DeletePusher(ctx context.Context, localpart, appID, pushKey string) error { + if _, err := s.db.ExecContext(ctx, s.q(deletePusherSQL), localpart, appID, pushKey); err != nil { + return fmt.Errorf("delete a pusher of %q: %w", localpart, err) + } + return nil +} + +// DeletePushersForDevices implements storage.PushStore. +func (s *Push) DeletePushersForDevices(ctx context.Context, localpart string, deviceIDs []string) error { + query, args := deleteAllPushersSQL, []any{localpart} + if len(deviceIDs) > 0 { + query += ` AND device_id IN (` + placeholders(len(deviceIDs)) + `)` + for _, id := range deviceIDs { + args = append(args, id) + } + } + if _, err := s.db.ExecContext(ctx, s.q(query), args...); err != nil { + return fmt.Errorf("delete the pushers of %q's devices: %w", localpart, err) + } + return nil +} + +// The sums are cast back to BIGINT because PostgreSQL widens SUM(BIGINT) +// to NUMERIC, which does not scan into an int64. +const ( + insertNotificationSQL = `INSERT INTO push_notifications + (id, user_id, room_id, event_id, event_pos, ts_ms, highlight, actions, is_read) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + + bumpRoomCountsSQL = `INSERT INTO push_room_counts + (user_id, room_id, notification_count, highlight_count, stream_pos) + VALUES (?, ?, 1, ?, ?) + ON CONFLICT (user_id, room_id) DO UPDATE SET + notification_count = push_room_counts.notification_count + 1, + highlight_count = push_room_counts.highlight_count + excluded.highlight_count, + stream_pos = excluded.stream_pos` + + markReadSQL = `UPDATE push_notifications SET is_read = ? + WHERE user_id = ? AND room_id = ? AND is_read = ? AND event_pos <= ?` + + countUnreadSQL = `SELECT COUNT(*), CAST(COALESCE(SUM(CASE WHEN highlight THEN 1 ELSE 0 END), 0) AS BIGINT) + FROM push_notifications WHERE user_id = ? AND room_id = ? AND is_read = ?` + + setRoomCountsSQL = `INSERT INTO push_room_counts + (user_id, room_id, notification_count, highlight_count, stream_pos) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (user_id, room_id) DO UPDATE SET + notification_count = excluded.notification_count, + highlight_count = excluded.highlight_count, + stream_pos = excluded.stream_pos` + + selectCountChangesSQL = `SELECT room_id FROM push_room_counts + WHERE user_id = ? AND stream_pos > ? AND stream_pos <= ? ORDER BY room_id` + + selectUnreadTotalSQL = `SELECT CAST(COALESCE(SUM(notification_count), 0) AS BIGINT) + FROM push_room_counts WHERE user_id = ?` + + notificationColumns = `id, user_id, room_id, event_id, event_pos, ts_ms, highlight, actions, is_read` + + pruneNotificationsSQL = `DELETE FROM push_notifications WHERE is_read = ? AND ts_ms < ?` +) + +// AddNotifications implements storage.PushStore. +// +// Every notification takes a position of its own, which is both its ID +// and the stamp on its room's counts: a sync that has seen a position has +// seen every count change at or below it. +func (s *Push) AddNotifications(ctx context.Context, ns []storage.NewNotification) (int64, error) { + if len(ns) == 0 { + return currentSequence(ctx, s.db, s.q, pushSequences, seqNotification) + } + var last int64 + err := s.tx(ctx, func(tx *sql.Tx) error { + for _, n := range ns { + pos, err := nextSequence(ctx, tx, s.q, pushSequences, seqNotification) + if err != nil { + return err + } + if _, err := tx.ExecContext(ctx, s.q(insertNotificationSQL), pos, n.UserID, n.RoomID, n.EventID, + n.EventPos, n.TimestampMS, n.Highlight, string(n.Actions), false); err != nil { + return fmt.Errorf("write a notification for %q: %w", n.UserID, err) + } + highlight := 0 + if n.Highlight { + highlight = 1 + } + if _, err := tx.ExecContext(ctx, s.q(bumpRoomCountsSQL), n.UserID, n.RoomID, highlight, pos); err != nil { + return fmt.Errorf("count a notification for %q: %w", n.UserID, err) + } + last = pos + } + return nil + }) + return last, err +} + +// MarkNotificationsRead implements storage.PushStore. +// +// The counts are recounted from the rows that are still unread rather than +// decremented, so that whatever got them out of step — a crash between +// two writes of an older version, a row pruned by hand — is put right by +// the next receipt. +func (s *Push) MarkNotificationsRead(ctx context.Context, userID, roomID string, upTo int64) (int64, bool, error) { + var ( + pos int64 + moved bool + ) + err := s.tx(ctx, func(tx *sql.Tx) error { + res, err := tx.ExecContext(ctx, s.q(markReadSQL), true, userID, roomID, false, upTo) + if err != nil { + return fmt.Errorf("mark %q's notifications read: %w", userID, err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("mark %q's notifications read: %w", userID, err) + } + if n == 0 { + pos, err = currentSequence(ctx, tx, s.q, pushSequences, seqNotification) + return err + } + + var notifications, highlights int64 + if err := tx.QueryRowContext(ctx, s.q(countUnreadSQL), userID, roomID, false). + Scan(¬ifications, &highlights); err != nil { + return fmt.Errorf("count %q's unread notifications: %w", userID, err) + } + if pos, err = nextSequence(ctx, tx, s.q, pushSequences, seqNotification); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, s.q(setRoomCountsSQL), userID, roomID, notifications, highlights, pos); err != nil { + return fmt.Errorf("write %q's notification counts: %w", userID, err) + } + moved = true + return nil + }) + return pos, moved, err +} + +// NotificationCounts implements storage.PushStore. +func (s *Push) NotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]storage.NotificationCounts, error) { + out := make(map[string]storage.NotificationCounts, len(roomIDs)) + if len(roomIDs) == 0 { + return out, nil + } + for _, id := range roomIDs { + out[id] = storage.NotificationCounts{} + } + + query := `SELECT room_id, notification_count, highlight_count FROM push_room_counts + WHERE user_id = ? AND room_id IN (` + placeholders(len(roomIDs)) + `)` + args := make([]any, 0, len(roomIDs)+1) + args = append(args, userID) + for _, id := range roomIDs { + args = append(args, id) + } + rows, err := s.db.QueryContext(ctx, s.q(query), args...) + if err != nil { + return nil, fmt.Errorf("read %q's notification counts: %w", userID, err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var roomID string + var c storage.NotificationCounts + if err := rows.Scan(&roomID, &c.Notifications, &c.Highlights); err != nil { + return nil, fmt.Errorf("read %q's notification counts: %w", userID, err) + } + out[roomID] = c + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read %q's notification counts: %w", userID, err) + } + return out, nil +} + +// NotificationCountChanges implements storage.PushStore. +func (s *Push) NotificationCountChanges(ctx context.Context, userID string, from, to int64) ([]string, error) { + rows, err := s.db.QueryContext(ctx, s.q(selectCountChangesSQL), userID, from, to) + if err != nil { + return nil, fmt.Errorf("read %q's notification count changes: %w", userID, err) + } + defer func() { _ = rows.Close() }() + var out []string + for rows.Next() { + var roomID string + if err := rows.Scan(&roomID); err != nil { + return nil, fmt.Errorf("read %q's notification count changes: %w", userID, err) + } + out = append(out, roomID) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read %q's notification count changes: %w", userID, err) + } + return out, nil +} + +// UnreadNotifications implements storage.PushStore. +func (s *Push) UnreadNotifications(ctx context.Context, userID string) (int, error) { + var n int64 + if err := s.db.QueryRowContext(ctx, s.q(selectUnreadTotalSQL), userID).Scan(&n); err != nil { + return 0, fmt.Errorf("count %q's unread notifications: %w", userID, err) + } + return int(n), nil +} + +// Notifications implements storage.PushStore. +func (s *Push) Notifications(ctx context.Context, userID string, before int64, limit int, onlyHighlight bool) ([]storage.Notification, error) { + query := `SELECT ` + notificationColumns + ` FROM push_notifications WHERE user_id = ?` + args := []any{userID} + if before > 0 { + query += ` AND id < ?` + args = append(args, before) + } + if onlyHighlight { + query += ` AND highlight = ?` + args = append(args, true) + } + query += ` ORDER BY id DESC LIMIT ?` + args = append(args, limit) + + rows, err := s.db.QueryContext(ctx, s.q(query), args...) + if err != nil { + return nil, fmt.Errorf("read %q's notifications: %w", userID, err) + } + defer func() { _ = rows.Close() }() + var out []storage.Notification + for rows.Next() { + var n storage.Notification + var actions string + if err := rows.Scan(&n.ID, &n.UserID, &n.RoomID, &n.EventID, &n.EventPos, &n.TimestampMS, + &n.Highlight, &actions, &n.Read); err != nil { + return nil, fmt.Errorf("read %q's notifications: %w", userID, err) + } + n.Actions = []byte(actions) + out = append(out, n) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read %q's notifications: %w", userID, err) + } + return out, nil +} + +// PruneNotifications implements storage.PushStore. +func (s *Push) PruneNotifications(ctx context.Context, beforeMS int64) (int64, error) { + res, err := s.db.ExecContext(ctx, s.q(pruneNotificationsSQL), true, beforeMS) + if err != nil { + return 0, fmt.Errorf("prune read notifications: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("prune read notifications: %w", err) + } + return n, nil +} diff --git a/server/internal/storage/sqlstore/sqlstore.go b/server/internal/storage/sqlstore/sqlstore.go index 2ee4725..f1882e4 100644 --- a/server/internal/storage/sqlstore/sqlstore.go +++ b/server/internal/storage/sqlstore/sqlstore.go @@ -62,6 +62,7 @@ type Store struct { fed *Federation media *Media keys *Keys + push *Push } // New builds a Store. It does not touch the database. @@ -83,6 +84,7 @@ func New(opts Options) (*Store, error) { fed: NewFederation(opts.DB, opts.Rebind), media: NewMedia(opts.DB, opts.Rebind, opts.IsUniqueViolation), keys: NewKeys(opts.DB, opts.Rebind), + push: NewPush(opts.DB, opts.Rebind), }, nil } @@ -119,6 +121,9 @@ func (s *Store) Media() storage.MediaStore { return s.media } // Keys implements storage.Database. func (s *Store) Keys() storage.KeyStore { return s.keys } +// Push implements storage.Database. +func (s *Store) Push() storage.PushStore { return s.push } + // Migrate implements storage.Database. func (s *Store) Migrate(ctx context.Context) error { ms, err := s.migrations() diff --git a/server/internal/storage/sqlstore/sync.go b/server/internal/storage/sqlstore/sync.go index b69feea..3959a49 100644 --- a/server/internal/storage/sqlstore/sync.go +++ b/server/internal/storage/sqlstore/sync.go @@ -115,6 +115,11 @@ func (s *Sync) StreamPositions(ctx context.Context) (storage.StreamPositions, er if err != nil { return storage.StreamPositions{}, err } + // And the push store's notification counts, for the same reason. + out.Notification, err = currentSequence(ctx, s.db, s.q, pushSequences, seqNotification) + if err != nil { + return storage.StreamPositions{}, err + } return out, nil } diff --git a/server/internal/storage/storage.go b/server/internal/storage/storage.go index 8e30bf0..1599895 100644 --- a/server/internal/storage/storage.go +++ b/server/internal/storage/storage.go @@ -84,6 +84,10 @@ type Database interface { // stream and the server-side key backup. Keys() KeyStore + // Push is the push store: pushers, and the notifications push rules + // produce with their per-room unread counts. + Push() PushStore + // Close releases the connection pool. Close() error } diff --git a/server/internal/storage/sync.go b/server/internal/storage/sync.go index 2c4fb9a..696f59c 100644 --- a/server/internal/storage/sync.go +++ b/server/internal/storage/sync.go @@ -32,6 +32,10 @@ type StreamPositions struct { // DeviceList is the keyserver's device list stream: the users whose // end-to-end keys a client should fetch again. DeviceList int64 + + // Notification is the push store's notification count stream: the + // rooms whose unread counts moved for a user. + Notification int64 } // TimelineDirection is which way a timeline read walks the stream. diff --git a/server/internal/syncapi/notifications.go b/server/internal/syncapi/notifications.go new file mode 100644 index 0000000..976959e --- /dev/null +++ b/server/internal/syncapi/notifications.go @@ -0,0 +1,51 @@ +package syncapi + +import ( + "context" + "slices" +) + +// pushRulesType is the account data push rules travel as. The push server +// renders it rather than the account data store, because a user who never +// changed a rule has none stored and still has the server defaults. +const pushRulesType = "m.push_rules" + +// UnreadNotifications is a joined room's unread counts for the user +// syncing. +type UnreadNotifications struct { + NotificationCount int `json:"notification_count"` + HighlightCount int `json:"highlight_count"` +} + +// unreadNotifications fills in the counts of every joined room in the +// response, in one read. +// +// Every joined room reported gets its counts, not only the ones whose +// counts moved: a room is in a response because something in it changed, +// and a client showing a new message wants the count beside it whether or +// not this particular sync is the one that moved it. +func (s *API) unreadNotifications(ctx context.Context, userID string, resp *Response) error { + if s.push == nil || len(resp.Rooms.Join) == 0 { + return nil + } + roomIDs := make([]string, 0, len(resp.Rooms.Join)) + for roomID := range resp.Rooms.Join { + roomIDs = append(roomIDs, roomID) + } + slices.Sort(roomIDs) + + counts, err := s.push.NotificationCounts(ctx, userID, roomIDs) + if err != nil { + return err + } + for _, roomID := range roomIDs { + room := resp.Rooms.Join[roomID] + c := counts[roomID] + room.UnreadNotifications = &UnreadNotifications{ + NotificationCount: c.Notifications, + HighlightCount: c.Highlights, + } + resp.Rooms.Join[roomID] = room + } + return nil +} diff --git a/server/internal/syncapi/receipts.go b/server/internal/syncapi/receipts.go index 4a01c8b..1bd4d8b 100644 --- a/server/internal/syncapi/receipts.go +++ b/server/internal/syncapi/receipts.go @@ -97,6 +97,11 @@ func (s *API) SetReceipt(ctx context.Context, req ReceiptRequest) error { if !moved { return nil } + // A receipt is also the user saying how far they have read, which is + // what clears their unread notifications. + if s.push != nil && s.isLocal(req.User) { + s.push.ReceiptSet(ctx, req.User, req.RoomID, req.ReceiptType, req.ThreadID, req.EventID) + } // A private receipt concerns only its sender; a public one is part of // the room's ephemeral stream and everybody in it wants it. if req.ReceiptType == ReceiptReadPrivate { diff --git a/server/internal/syncapi/sync.go b/server/internal/syncapi/sync.go index 9ea2af1..010e45b 100644 --- a/server/internal/syncapi/sync.go +++ b/server/internal/syncapi/sync.go @@ -97,6 +97,12 @@ type JoinedRoom struct { Timeline Timeline `json:"timeline"` Ephemeral EphemeralEventList `json:"ephemeral"` AccountData AccountDataList `json:"account_data"` + + // UnreadNotifications is the user's unread notification counts in the + // room. It is a pointer so that a server without a push server leaves + // it out rather than claiming zero: an absent count says nothing, + // where a zero says "you have read everything". + UnreadNotifications *UnreadNotifications `json:"unread_notifications,omitempty"` } // InvitedRoom is a room the user has been invited to. It carries no @@ -327,8 +333,11 @@ func (s *API) build(ctx context.Context, req SyncRequest, filter Filter, since T if err := s.buildRooms(ctx, req, filter, since, to, rooms, &resp); err != nil { return Response{}, false, err } + if err := s.unreadNotifications(ctx, req.User, &resp); err != nil { + return Response{}, false, err + } - accountData, err := s.globalAccountData(ctx, req.Localpart, since.AccountData, to.AccountData, filter) + accountData, err := s.globalAccountData(ctx, req, since.AccountData, to.AccountData, filter) if err != nil { return Response{}, false, err } @@ -537,6 +546,22 @@ func (s *API) changedRooms(ctx context.Context, req SyncRequest, filter Filter, } } + // A room whose unread counts moved without an event in this window — + // read on another device, or counted after the event was already sent + // to this client — is reported so that the client hears the new + // counts. + if s.push != nil && to.Notification > since.Notification { + changed, err := s.push.NotificationCountChanges(ctx, req.User, since.Notification, to.Notification) + if err != nil { + return nil, err + } + for _, roomID := range changed { + if slices.Contains(joined, roomID) { + out[roomID] = struct{}{} + } + } + } + for _, room := range rooms { if room.Position > since.PDU && room.Position <= to.PDU { out[room.RoomID] = struct{}{} @@ -640,12 +665,20 @@ func (s *API) roomEphemeral(ctx context.Context, userID, roomID string, window r } // globalAccountData reads the account data that is not about a room. -func (s *API) globalAccountData(ctx context.Context, localpart string, from, to int64, filter Filter) ([]AccountDataEvent, error) { - entries, err := s.accountDataEntries(ctx, localpart, from, to) +func (s *API) globalAccountData(ctx context.Context, req SyncRequest, from, to int64, filter Filter) ([]AccountDataEvent, error) { + entries, err := s.accountDataEntries(ctx, req.Localpart, from, to) if err != nil { return nil, err } - return s.accountDataEvents(ctx, localpart, entries, "", filter.AccountDataFilter()) + // Every user has push rules, written or not: a user who has never + // changed one has the server defaults, and a client reads its rules + // from the account data of its first sync whichever it is. + if from == 0 && s.push != nil && !slices.ContainsFunc(entries, func(e storage.AccountDataEntry) bool { + return e.RoomID == "" && e.DataType == pushRulesType + }) { + entries = append(entries, storage.AccountDataEntry{DataType: pushRulesType}) + } + return s.accountDataEvents(ctx, req, entries, "", filter.AccountDataFilter()) } // roomAccountData reads one room's account data. @@ -655,7 +688,7 @@ func (s *API) roomAccountData(ctx context.Context, localpart, roomID string, win if err != nil { return nil, err } - return s.accountDataEvents(ctx, localpart, entries, roomID, filter.RoomAccountDataFilter()) + return s.accountDataEvents(ctx, SyncRequest{Localpart: localpart}, entries, roomID, filter.RoomAccountDataFilter()) } // accountDataEntries lists which of a user's account data changed in a @@ -669,7 +702,7 @@ func (s *API) accountDataEntries(ctx context.Context, localpart string, from, to // accountDataEvents turns the entries of one scope — global, or one room — // into the events a sync reports. -func (s *API) accountDataEvents(ctx context.Context, localpart string, entries []storage.AccountDataEntry, roomID string, filter storage.TimelineFilter) ([]AccountDataEvent, error) { +func (s *API) accountDataEvents(ctx context.Context, req SyncRequest, entries []storage.AccountDataEntry, roomID string, filter storage.TimelineFilter) ([]AccountDataEvent, error) { out := []AccountDataEvent{} for _, entry := range entries { if entry.RoomID != roomID { @@ -678,7 +711,15 @@ func (s *API) accountDataEvents(ctx context.Context, localpart string, entries [ if !matchesEventFilter(entry.DataType, "", filter) { continue } - content, err := s.accounts.AccountData(ctx, localpart, entry.RoomID, entry.DataType) + if entry.RoomID == "" && entry.DataType == pushRulesType && s.push != nil { + content, err := s.push.Document(ctx, req.Localpart, req.User) + if err != nil { + return nil, err + } + out = append(out, AccountDataEvent{Type: entry.DataType, Content: content}) + continue + } + content, err := s.accounts.AccountData(ctx, req.Localpart, entry.RoomID, entry.DataType) switch { case errors.Is(err, storage.ErrNoSuchAccountData): // Deleted between the two reads: nothing to report. diff --git a/server/internal/syncapi/syncapi.go b/server/internal/syncapi/syncapi.go index 2599362..f6f1445 100644 --- a/server/internal/syncapi/syncapi.go +++ b/server/internal/syncapi/syncapi.go @@ -28,8 +28,9 @@ // // Phase 4 of PLAN.md, with device lists from Phase 7: the keyserver writes // the device list stream, and this package works out which of its changes -// concern whom (devicelists.go). What it does not do yet is notification -// counts, which need push rules (Phase 8); docs/sync.md names that gap. +// concern whom (devicelists.go). Notification counts, from Phase 8, are +// the push server's: it writes them and their stream, and a sync reports +// them per joined room (notifications.go). package syncapi import ( @@ -82,12 +83,24 @@ type Keys interface { DeviceListChanges(ctx context.Context, userIDs []string, from, to int64) ([]storage.DeviceListChange, error) } +// Push is the part of the push server a sync reads: the push rules it +// reports as account data, the unread counts of each room, and which +// rooms' counts moved. It is also told about read receipts, which are what +// clear the counts. *pushserver.API satisfies it. +type Push interface { + Document(ctx context.Context, localpart, userID string) ([]byte, error) + NotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]storage.NotificationCounts, error) + NotificationCountChanges(ctx context.Context, userID string, from, to int64) ([]string, error) + ReceiptSet(ctx context.Context, userID, roomID, receiptType, threadID, eventID string) +} + // API is the syncapi component. type API struct { serverName string rooms *roomserver.API accounts Accounts keys Keys + push Push store storage.SyncStore notifier *Notifier typing *typingTracker @@ -126,6 +139,10 @@ type Options struct { // both device lists empty. Keys Keys + // Push is the push server. Nil leaves the notification counts out of + // a sync, and m.push_rules is then whatever account data holds. + Push Push + // Store is the sync half of storage. Store storage.SyncStore @@ -206,6 +223,7 @@ func New(opts Options) (*API, error) { rooms: opts.Rooms, accounts: opts.Accounts, keys: opts.Keys, + push: opts.Push, store: opts.Store, notifier: opts.Notifier, typing: newTypingTracker(opts.TypingTimeout, opts.Notifier, opts.Now), diff --git a/server/internal/syncapi/token.go b/server/internal/syncapi/token.go index 405a282..662cec3 100644 --- a/server/internal/syncapi/token.go +++ b/server/internal/syncapi/token.go @@ -18,12 +18,13 @@ import ( // // The wire form is // -// s{pdu}_{receipt}_{to_device}_{account_data}_{presence}_{device_list} +// s{pdu}_{receipt}_{to_device}_{account_data}_{presence}_{device_list}_{notification} // // with trailing zeroes left off, so a client that has only ever seen room -// events carries "s12" rather than "s12_0_0_0_0_0". The device list -// position came with Phase 7 and was added at the end, which is what lets -// every token handed out before it go on parsing. The leading "s" is there +// events carries "s12" rather than "s12_0_0_0_0_0_0". The device list +// position came with Phase 7 and the notification count position with +// Phase 8, each added at the end, which is what lets every token handed +// out before them go on parsing. The leading "s" is there // to make a token recognisable in a log and to leave room for a second // token shape later — Simplified Sliding Sync (PLAN.md phase 10) has its // own — without any ambiguity about which is which. @@ -38,10 +39,13 @@ type Token struct { AccountData int64 Presence int64 DeviceList int64 + + // Notification is the push server's notification count stream. + Notification int64 } // tokenPositions is how many positions a token carries. -const tokenPositions = 6 +const tokenPositions = 7 // tokenPrefix marks a classic /sync token. const tokenPrefix = "s" @@ -52,7 +56,7 @@ func (t Token) IsInitial() bool { return t == Token{} } // String renders the wire form. func (t Token) String() string { - parts := []int64{t.PDU, t.Receipt, t.ToDevice, t.AccountData, t.Presence, t.DeviceList} + parts := []int64{t.PDU, t.Receipt, t.ToDevice, t.AccountData, t.Presence, t.DeviceList, t.Notification} // Trailing zeroes carry no information, and leaving them off keeps the // common token short. last := 0 @@ -117,6 +121,8 @@ func ParseToken(raw string) (Token, error) { AccountData: positions[3], Presence: positions[4], DeviceList: positions[5], + + Notification: positions[6], }, nil } @@ -130,6 +136,8 @@ func tokenAt(positions storage.StreamPositions) Token { AccountData: positions.AccountData, Presence: positions.Presence, DeviceList: positions.DeviceList, + + Notification: positions.Notification, } } diff --git a/server/internal/syncapi/token_test.go b/server/internal/syncapi/token_test.go index 23b0e16..faca98a 100644 --- a/server/internal/syncapi/token_test.go +++ b/server/internal/syncapi/token_test.go @@ -17,6 +17,12 @@ func TestTokenRoundTrips(t *testing.T) { {"events and receipts", syncapi.Token{PDU: 12, Receipt: 3}, "s12_3"}, { "every stream", + syncapi.Token{PDU: 1, Receipt: 2, ToDevice: 3, AccountData: 4, Presence: 5, DeviceList: 6, Notification: 7}, + "s1_2_3_4_5_6_7", + }, + { + // A Phase 7 token, from before the notification stream. + "six streams", syncapi.Token{PDU: 1, Receipt: 2, ToDevice: 3, AccountData: 4, Presence: 5, DeviceList: 6}, "s1_2_3_4_5_6", }, @@ -81,13 +87,13 @@ func TestParseTokenAcceptsAShortToken(t *testing.T) { func TestParseTokenRejectsNonsense(t *testing.T) { for _, raw := range []string{ - "nonsense", // no prefix - "t1_2", // a prefix this build does not write - "s", // a prefix and nothing else - "s1_-1", // a negative position outside the event stream - "sx", // not a number - "s1_", // a trailing separator - "s1_2_3_4_5_6_7", // more positions than there are streams + "nonsense", // no prefix + "t1_2", // a prefix this build does not write + "s", // a prefix and nothing else + "s1_-1", // a negative position outside the event stream + "sx", // not a number + "s1_", // a trailing separator + "s1_2_3_4_5_6_7_8", // more positions than there are streams "s1.5", } { if _, err := syncapi.ParseToken(raw); err == nil { diff --git a/server/internal/userapi/userapi.go b/server/internal/userapi/userapi.go index 21ec3ee..603fb41 100644 --- a/server/internal/userapi/userapi.go +++ b/server/internal/userapi/userapi.go @@ -8,8 +8,9 @@ // without starting a server. // // Nothing in this package knows about rooms. A device's encryption keys -// are the keyserver's, which hears about a device going away through the -// DeviceObserver; its push rules arrive in Phase 8. +// are the keyserver's and its pushers the push server's, and both hear +// about a device going away through the DeviceObserver. Push rules are +// account data, but the push server is their only writer. package userapi import ( diff --git a/server/test/complement/run.sh b/server/test/complement/run.sh index d69ce51..9a7060c 100755 --- a/server/test/complement/run.sh +++ b/server/test/complement/run.sh @@ -25,14 +25,14 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" # # csapi: the Client-Server tests, with federation off. Phase 4 added the # stream tests: /sync in all its forms, /messages, /context, filters, -# typing, receipts and to-device messaging. What is still out is anything -# needing a later phase — the notification counts of the push rules -# (Phase 8). Phase 6 added the media tests: uploads, async uploads, -# downloads with their file names and thumbnails. The remote-media tests -# are in the federation set, which TestRemote.* already covers. Phase 7 -# added the keyserver's: key uploads, claims, changes, device lists, -# fallback keys, cross-signing and key backup. -CSAPI_TESTS='TestUploadKey.*|TestKeyClaimOrdering|TestKeyChanges.*|TestDeviceListUpdates.*|TestE2EKeyBackup.*|TestFallbackKeys.*|TestCrossSigning.*|TestMedia.*|TestContent.*|TestLocalPngThumbnail|TestAsyncUpload.*|TestUploadLimit|TestRegistration|TestLogin|TestDeviceManagement|TestRoomCreate|TestRoomState|TestRoomAlias|TestRoomForget|TestMembership.*|TestRedaction.*|TestJoinRoom.*|TestLeaveRoom.*|TestKickRoom.*|TestBanRoom.*|TestPublicRoom.*|TestSync.*|TestMessages.*|TestContext.*|TestFilter.*|TestTyping.*|TestReceipt.*|TestToDevice.*|TestRoomCreationReportsEventsToMyself|TestLazyLoading.*|TestWriteMDirectAccountData' +# typing, receipts and to-device messaging. Phase 6 added the media +# tests: uploads, async uploads, downloads with their file names and +# thumbnails. The remote-media tests are in the federation set, which +# TestRemote.* already covers. Phase 7 added the keyserver's: key uploads, +# claims, changes, device lists, fallback keys, cross-signing and key +# backup. Phase 8 added the push rule tests: the rules API, and rule +# changes arriving through /sync. +CSAPI_TESTS='TestPushRuleCacheHealth|TestPushSync|TestUploadKey.*|TestKeyClaimOrdering|TestKeyChanges.*|TestDeviceListUpdates.*|TestE2EKeyBackup.*|TestFallbackKeys.*|TestCrossSigning.*|TestMedia.*|TestContent.*|TestLocalPngThumbnail|TestAsyncUpload.*|TestUploadLimit|TestRegistration|TestLogin|TestDeviceManagement|TestRoomCreate|TestRoomState|TestRoomAlias|TestRoomForget|TestMembership.*|TestRedaction.*|TestJoinRoom.*|TestLeaveRoom.*|TestKickRoom.*|TestBanRoom.*|TestPublicRoom.*|TestSync.*|TestMessages.*|TestContext.*|TestFilter.*|TestTyping.*|TestReceipt.*|TestToDevice.*|TestRoomCreationReportsEventsToMyself|TestLazyLoading.*|TestWriteMDirectAccountData' # federation: Phase 5. Joins, leaves and invites across servers, keys, # transactions, missing events, server ACLs and the queries. Partial-state diff --git a/warp.yaml.example b/warp.yaml.example index ec76274..b43d2dc 100644 --- a/warp.yaml.example +++ b/warp.yaml.example @@ -220,6 +220,27 @@ media: # Matrix 1.11. WARP_MEDIA_LEGACY_UNAUTHENTICATED_DOWNLOADS legacy_unauthenticated_downloads: false +push: + # Send pushes to the push gateways of users' pushers — what wakes a + # phone for a new message. Off, notifications are still counted (unread + # badges come from the counts, not from pushes) and nothing leaves the + # server. WARP_PUSH_ENABLED + enabled: true + + # Put the event's content into a push. Off, a push says who sent what + # kind of event in which room and nothing of what it said, for when + # message text should not pass through a third-party push service. + # WARP_PUSH_INCLUDE_CONTENT + include_content: true + + # One request to a push gateway. WARP_PUSH_GATEWAY_TIMEOUT + gateway_timeout: 10s + + # How long a notification that has been read is kept for GET + # /notifications. Unread ones are kept until read. + # WARP_PUSH_NOTIFICATION_RETENTION + notification_retention: 720h + registration: # Off by default. Make the first account with # `warp create-account --username you --admin`, then turn registration