Phase 8: push and notifications

Implements PLAN.md Phase 8. internal/pushrules is the rule language: the
spec's default ruleset, all six condition kinds, glob and word-boundary
matching, intentional mentions switching off the legacy body rules, and
rule ordering as Synapse has it. internal/pushserver keeps the rules,
pushers and notifications and talks to push gateways.

Push rules are the m.push_rules account data, written only by the push
server; reads keep the user's own rules and the enabled/actions of each
default and take the rest from the current defaults, so new default rules
reach every user without a migration. PUT account_data/m.push_rules is
refused with 405.

The push server listens on the roomserver's emit stage and evaluates each
event on its own worker for every local member and an invite's target.
Notifications and per-room unread counts are stored in migration 0010;
counts are a seventh sync token stream, so /sync reports
unread_notifications and picks up count changes made after the event or by
a receipt from another device. m.read and m.read.private receipts clear
the counts; leaving a room clears it.

Pushers (http kind only) push through a bounded sender pool with retries,
honour event_id_only, drop push keys the gateway rejects, and go away with
their device. GET /notifications pages the history. New config section
push: enabled, include_content, gateway_timeout, notification_retention.

Account data writes now wake the user's blocked syncs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

server/internal/clientapi/accountdata.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/dariuszjaniak/warp/server/internal/httputil"
+	"github.com/dariuszjaniak/warp/server/internal/pushserver"
 	"github.com/dariuszjaniak/warp/server/internal/userapi"
 )
 
@@ -48,7 +49,14 @@ func (c *ClientAPI) getAccountData(w http.ResponseWriter, r *http.Request, auth
 		return httputil.MissingParam("An account data type is required")
 	}
 
-	content, err := c.users.AccountData(r.Context(), localpart, roomID, dataType)
+	var content []byte
+	if roomID == "" && dataType == pushserver.AccountDataType && c.push != nil {
+		// The push server's, rendered over the defaults: a user who never
+		// changed a rule has nothing stored and still has rules.
+		content, err = c.push.Document(r.Context(), localpart, auth.UserID)
+	} else {
+		content, err = c.users.AccountData(r.Context(), localpart, roomID, dataType)
+	}
 	switch {
 	case errors.Is(err, userapi.ErrNoSuchAccountData):
 		return httputil.NotFound("Account data not found")
@@ -93,6 +101,10 @@ func (c *ClientAPI) putAccountData(w http.ResponseWriter, r *http.Request, auth
 	if dataType == typeFullyRead {
 		return httputil.BadJSON("%s may only be set through /rooms/{roomId}/read_markers", typeFullyRead)
 	}
+	if dataType == pushserver.AccountDataType {
+		return httputil.NewError(http.StatusMethodNotAllowed, httputil.ErrCodeBadJSON,
+			"%s may only be changed through the /pushrules endpoints", pushserver.AccountDataType)
+	}
 
 	// The content is stored as the client sent it, so it is read as raw
 	// JSON and only checked for being a JSON object at all.
@@ -114,6 +126,11 @@ func (c *ClientAPI) putAccountData(w http.ResponseWriter, r *http.Request, auth
 	if err := c.users.SetAccountData(r.Context(), localpart, roomID, dataType, content); err != nil {
 		return err
 	}
+	// Account data reaches a client's other devices through /sync, and
+	// one of them blocked in a long poll should hear about it now.
+	if c.sync != nil {
+		c.sync.Notifier().NotifyUser(auth.UserID)
+	}
 	httputil.WriteJSON(w, http.StatusOK, struct{}{})
 	return nil
 }

server/internal/clientapi/clientapi.go

@@ -22,6 +22,7 @@ import (
 	"github.com/dariuszjaniak/warp/server/internal/httputil"
 	"github.com/dariuszjaniak/warp/server/internal/keyserver"
 	"github.com/dariuszjaniak/warp/server/internal/mediaapi"
+	"github.com/dariuszjaniak/warp/server/internal/pushserver"
 	"github.com/dariuszjaniak/warp/server/internal/ratelimit"
 	"github.com/dariuszjaniak/warp/server/internal/roomserver"
 	"github.com/dariuszjaniak/warp/server/internal/settings"
@@ -38,6 +39,7 @@ type ClientAPI struct {
 	sync          *syncapi.API
 	keys          *keyserver.API
 	media         *mediaapi.API
+	push          *pushserver.API
 	legacyMedia   bool
 	settings      *settings.Service
 	federation    *federation.Gate
@@ -103,6 +105,10 @@ type Options struct {
 	// answer M_UNRECOGNIZED.
 	Media *mediaapi.API
 
+	// Push is the push server. Without it the push rule, pusher and
+	// notification endpoints answer M_UNRECOGNIZED.
+	Push *pushserver.API
+
 	// LegacyMediaDownloads serves the deprecated unauthenticated download
 	// and thumbnail endpoints too.
 	LegacyMediaDownloads bool
@@ -162,6 +168,7 @@ func New(opts Options) (*ClientAPI, error) {
 		sync:          opts.Sync,
 		keys:          opts.Keys,
 		media:         opts.Media,
+		push:          opts.Push,
 		legacyMedia:   opts.LegacyMediaDownloads,
 		settings:      opts.Settings,
 		federation:    opts.Federation,
@@ -257,6 +264,7 @@ func (c *ClientAPI) RegisterRoutes(r *httputil.Router) {
 	c.registerSyncRoutes(r)
 	c.registerKeyRoutes(r)
 	c.registerMediaRoutes(r)
+	c.registerPushRoutes(r)
 }
 
 // registerSyncRoutes adds the endpoints the syncapi serves, if there is

server/internal/clientapi/push.go

@@ -0,0 +1,303 @@
+package clientapi
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"strconv"
+
+	"github.com/dariuszjaniak/warp/server/internal/httputil"
+	"github.com/dariuszjaniak/warp/server/internal/pushrules"
+	"github.com/dariuszjaniak/warp/server/internal/pushserver"
+	"github.com/dariuszjaniak/warp/server/internal/userapi"
+)
+
+// The push endpoints: push rules, pushers and the notification list.
+// Push rules are also the m.push_rules account data, which a client may
+// read through the account data endpoints but not write through them.
+
+// pushRulesScope is the one scope Matrix has left. The spec once had a
+// "device" scope too; it was never implemented anywhere and has been
+// removed.
+const pushRulesScope = "global"
+
+// registerPushRoutes adds the push endpoints, if there is a push server.
+func (c *ClientAPI) registerPushRoutes(r *httputil.Router) {
+	if c.push == nil {
+		return
+	}
+	// The ruleset is at /pushrules/ with the slash; the form without it is
+	// served too, because clients send both.
+	c.authed(r, "GET "+v3+"/pushrules/{$}", c.limitDefault, c.handleGetPushRules)
+	c.authed(r, "GET "+v3+"/pushrules", c.limitDefault, c.handleGetPushRules)
+	c.authed(r, "GET "+v3+"/pushrules/{scope}/{$}", c.limitDefault, c.handleGetPushRuleScope)
+	c.authed(r, "GET "+v3+"/pushrules/{scope}", c.limitDefault, c.handleGetPushRuleScope)
+	c.authed(r, "GET "+v3+"/pushrules/{scope}/{kind}/{ruleID}", c.limitDefault, c.handleGetPushRule)
+	c.authed(r, "PUT "+v3+"/pushrules/{scope}/{kind}/{ruleID}", c.limitDefault, c.handlePutPushRule)
+	c.authed(r, "DELETE "+v3+"/pushrules/{scope}/{kind}/{ruleID}", c.limitDefault, c.handleDeletePushRule)
+	c.authed(r, "GET "+v3+"/pushrules/{scope}/{kind}/{ruleID}/{attr}", c.limitDefault, c.handleGetPushRuleAttr)
+	c.authed(r, "PUT "+v3+"/pushrules/{scope}/{kind}/{ruleID}/{attr}", c.limitDefault, c.handlePutPushRuleAttr)
+
+	c.authed(r, "GET "+v3+"/pushers", c.limitDefault, c.handleGetPushers)
+	c.authed(r, "POST "+v3+"/pushers/set", c.limitDefault, c.handleSetPusher)
+
+	c.authed(r, "GET "+v3+"/notifications", c.limitDefault, c.handleGetNotifications)
+}
+
+// handleGetPushRules implements GET /_matrix/client/v3/pushrules/.
+func (c *ClientAPI) handleGetPushRules(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
+	rs, err := c.push.Ruleset(r.Context(), auth.Localpart, auth.UserID)
+	if err != nil {
+		return err
+	}
+	httputil.WriteJSON(w, http.StatusOK, pushrules.Document{Global: rs})
+	return nil
+}
+
+// handleGetPushRuleScope implements GET /_matrix/client/v3/pushrules/global/.
+func (c *ClientAPI) handleGetPushRuleScope(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
+	if err := checkPushScope(r); err != nil {
+		return err
+	}
+	rs, err := c.push.Ruleset(r.Context(), auth.Localpart, auth.UserID)
+	if err != nil {
+		return err
+	}
+	httputil.WriteJSON(w, http.StatusOK, rs)
+	return nil
+}
+
+// handleGetPushRule implements GET /_matrix/client/v3/pushrules/global/{kind}/{ruleId}.
+func (c *ClientAPI) handleGetPushRule(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
+	kind, ruleID, err := pushRulePath(r)
+	if err != nil {
+		return err
+	}
+	rule, err := c.push.Rule(r.Context(), auth.Localpart, auth.UserID, kind, ruleID)
+	if err != nil {
+		return pushRuleError(err)
+	}
+	httputil.WriteJSON(w, http.StatusOK, rule)
+	return nil
+}
+
+// handlePutPushRule implements PUT /_matrix/client/v3/pushrules/global/{kind}/{ruleId}.
+func (c *ClientAPI) handlePutPushRule(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
+	kind, ruleID, err := pushRulePath(r)
+	if err != nil {
+		return err
+	}
+	var body pushrules.NewRule
+	if err := httputil.DecodeJSON(w, r, &body); err != nil {
+		return err
+	}
+	query := r.URL.Query()
+	err = c.push.PutRule(r.Context(), auth.Localpart, auth.UserID, kind, ruleID, body,
+		query.Get("before"), query.Get("after"))
+	if err != nil {
+		return pushRuleError(err)
+	}
+	httputil.WriteJSON(w, http.StatusOK, struct{}{})
+	return nil
+}
+
+// handleDeletePushRule implements DELETE /_matrix/client/v3/pushrules/global/{kind}/{ruleId}.
+func (c *ClientAPI) handleDeletePushRule(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
+	kind, ruleID, err := pushRulePath(r)
+	if err != nil {
+		return err
+	}
+	if err := c.push.DeleteRule(r.Context(), auth.Localpart, auth.UserID, kind, ruleID); err != nil {
+		return pushRuleError(err)
+	}
+	httputil.WriteJSON(w, http.StatusOK, struct{}{})
+	return nil
+}
+
+// The two attributes of a rule a client may read and write on their own.
+const (
+	pushRuleAttrEnabled = "enabled"
+	pushRuleAttrActions = "actions"
+)
+
+// handleGetPushRuleAttr implements GET .../{ruleId}/enabled and .../{ruleId}/actions.
+func (c *ClientAPI) handleGetPushRuleAttr(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
+	kind, ruleID, err := pushRulePath(r)
+	if err != nil {
+		return err
+	}
+	attr := r.PathValue("attr")
+	if attr != pushRuleAttrEnabled && attr != pushRuleAttrActions {
+		return httputil.Unrecognized()
+	}
+	rule, err := c.push.Rule(r.Context(), auth.Localpart, auth.UserID, kind, ruleID)
+	if err != nil {
+		return pushRuleError(err)
+	}
+	if attr == pushRuleAttrEnabled {
+		httputil.WriteJSON(w, http.StatusOK, map[string]bool{"enabled": rule.Enabled})
+		return nil
+	}
+	httputil.WriteJSON(w, http.StatusOK, map[string][]json.RawMessage{"actions": rule.Actions})
+	return nil
+}
+
+// handlePutPushRuleAttr implements PUT .../{ruleId}/enabled and .../{ruleId}/actions.
+func (c *ClientAPI) handlePutPushRuleAttr(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
+	kind, ruleID, err := pushRulePath(r)
+	if err != nil {
+		return err
+	}
+	switch r.PathValue("attr") {
+	case pushRuleAttrEnabled:
+		var body struct {
+			Enabled *bool `json:"enabled"`
+		}
+		if err := httputil.DecodeJSON(w, r, &body); err != nil {
+			return err
+		}
+		if body.Enabled == nil {
+			return httputil.MissingParam("enabled is required")
+		}
+		err = c.push.SetRuleEnabled(r.Context(), auth.Localpart, auth.UserID, kind, ruleID, *body.Enabled)
+	case pushRuleAttrActions:
+		var body struct {
+			Actions []json.RawMessage `json:"actions"`
+		}
+		if err := httputil.DecodeJSON(w, r, &body); err != nil {
+			return err
+		}
+		if body.Actions == nil {
+			return httputil.MissingParam("actions is required")
+		}
+		err = c.push.SetRuleActions(r.Context(), auth.Localpart, auth.UserID, kind, ruleID, body.Actions)
+	default:
+		return httputil.Unrecognized()
+	}
+	if err != nil {
+		return pushRuleError(err)
+	}
+	httputil.WriteJSON(w, http.StatusOK, struct{}{})
+	return nil
+}
+
+func checkPushScope(r *http.Request) error {
+	if scope := r.PathValue("scope"); scope != pushRulesScope {
+		return httputil.InvalidParam("%q is not a push rule scope; the only one is %q", scope, pushRulesScope)
+	}
+	return nil
+}
+
+// pushRulePath reads the scope, kind and rule ID of a push rule path.
+func pushRulePath(r *http.Request) (pushrules.Kind, string, error) {
+	if err := checkPushScope(r); err != nil {
+		return "", "", err
+	}
+	kind, ok := pushrules.ParseKind(r.PathValue("kind"))
+	if !ok {
+		return "", "", httputil.InvalidParam("%q is not a kind of push rule", r.PathValue("kind"))
+	}
+	ruleID := r.PathValue("ruleID")
+	if ruleID == "" {
+		return "", "", httputil.MissingParam("A rule ID is required")
+	}
+	return kind, ruleID, nil
+}
+
+// pushRuleError maps the push server's errors onto the Matrix ones.
+func pushRuleError(err error) error {
+	switch {
+	case errors.Is(err, pushserver.ErrNoSuchRule):
+		return httputil.NotFound("Push rule not found")
+	case errors.Is(err, pushserver.ErrNoSuchAnchor):
+		return httputil.NotFound("The rule named by before or after is not a rule of yours of this kind")
+	case errors.Is(err, pushserver.ErrInvalidRule):
+		return httputil.InvalidParam("%v", err)
+	}
+	return err
+}
+
+// handleGetPushers implements GET /_matrix/client/v3/pushers.
+func (c *ClientAPI) handleGetPushers(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
+	pushers, err := c.push.Pushers(r.Context(), auth.Localpart)
+	if err != nil {
+		return err
+	}
+	httputil.WriteJSON(w, http.StatusOK, map[string][]pushserver.Pusher{"pushers": pushers})
+	return nil
+}
+
+// handleSetPusher implements POST /_matrix/client/v3/pushers/set.
+func (c *ClientAPI) handleSetPusher(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
+	// kind is read raw, because it has three meanings rather than two:
+	// absent is a malformed request, null removes the pusher, and a
+	// string sets one.
+	var body struct {
+		pushserver.SetPusherRequest
+		Kind json.RawMessage `json:"kind"`
+	}
+	if err := httputil.DecodeJSON(w, r, &body); err != nil {
+		return err
+	}
+	req := body.SetPusherRequest
+	switch string(body.Kind) {
+	case "":
+		return httputil.MissingParam("kind is required")
+	case "null":
+		req.Kind = nil
+	default:
+		var kind string
+		if err := json.Unmarshal(body.Kind, &kind); err != nil {
+			return httputil.BadJSON("kind must be a string or null")
+		}
+		req.Kind = &kind
+	}
+
+	err := c.push.SetPusher(r.Context(), auth.Localpart, auth.DeviceID, req)
+	switch {
+	case errors.Is(err, pushserver.ErrMissingPusherField):
+		return httputil.MissingParam("%v", err)
+	case errors.Is(err, pushserver.ErrInvalidPusher):
+		return httputil.InvalidParam("%v", err)
+	case err != nil:
+		return err
+	}
+	httputil.WriteJSON(w, http.StatusOK, struct{}{})
+	return nil
+}
+
+// handleGetNotifications implements GET /_matrix/client/v3/notifications.
+func (c *ClientAPI) handleGetNotifications(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
+	query := r.URL.Query()
+	req := pushserver.NotificationsRequest{
+		UserID: auth.UserID,
+		From:   query.Get("from"),
+	}
+	if raw := query.Get("limit"); raw != "" {
+		limit, err := strconv.Atoi(raw)
+		if err != nil || limit < 1 {
+			return httputil.InvalidParam("limit must be a positive integer")
+		}
+		req.Limit = limit
+	}
+	switch only := query.Get("only"); only {
+	case "":
+	case "highlight":
+		req.OnlyHighlight = true
+	default:
+		return httputil.InvalidParam("only must be highlight")
+	}
+
+	notifications, next, err := c.push.Notifications(r.Context(), req)
+	if errors.Is(err, pushserver.ErrInvalidFrom) {
+		return httputil.InvalidParam("from is not a token this server handed out")
+	}
+	if err != nil {
+		return err
+	}
+	httputil.WriteJSON(w, http.StatusOK, struct {
+		Notifications []pushserver.Notification `json:"notifications"`
+		NextToken     string                    `json:"next_token,omitempty"`
+	}{notifications, next})
+	return nil
+}

server/internal/clientapi/sync_test.go

@@ -260,7 +260,7 @@ func TestSyncRejectsAnUnreadableToken(t *testing.T) {
 	// "s-1" is not here: a negative event position is a /messages token
 	// from inside backfilled history, and a valid ?since=. Only the event
 	// stream goes below zero.
-	for _, since := range []string{"nonsense", "s1_-1", "sx", "s1_2_3_4_5_6_7"} {
+	for _, since := range []string{"nonsense", "s1_-1", "sx", "s1_2_3_4_5_6_7_8"} {
 		rec := ts.get("/_matrix/client/v3/sync?since="+url.QueryEscape(since), alice.AccessToken)
 		expectError(t, rec, http.StatusBadRequest, "M_INVALID_PARAM")
 	}