Phase 10: hardening and ops

Implements PLAN.md Phase 10.

Simplified Sliding Sync (MSC4186) is served at the unstable
org.matrix.simplified_msc3575 endpoint and advertised in /versions. Lists
are sorted by activity and filtered by DM, invite, encryption, room type,
space and tag; room subscriptions, required_state with wildcards, $ME and
$LAZY, and the to_device, e2ee, account_data, receipts and typing
extensions are supported. Per-connection state is kept in memory and an
unknown pos answers M_UNKNOWN_POS.

Threads and relations: m.relates_to is indexed on roomserver_events
(migration 0012), /relations (with recursion) and /threads are served, and
every event a timeline read returns carries its m.thread, m.replace and
m.reference summaries in unsigned.m.relations. Notifications carry a
thread ID (migration 0013): threaded receipts clear only their thread, and
sync reports unread_thread_notifications when the filter asks.

Spaces: /hierarchy walks a space breadth first, asking other servers about
rooms this one is not in through the new federation hierarchy endpoint;
room_summary describes one room. /search runs on FTS5 (sqlite, pocketbase)
or tsvector (postgres), bounded to what each user may see, with context,
profiles, state, grouping and paging.

The PocketBase driver is real: the sqlite schema and store on an embedded
PocketBase app's database, with media in its filesystem, local or S3
(database.pocketbase.s3). It runs the full storage test suite.

Ops: OpenTelemetry tracing over OTLP/HTTP (tracing.*), with server spans
per route, component spans, and trace propagation on federation, push and
application service calls. New metrics for persisted events, waiting
syncs, sliding sync connections and the database pool, with a Grafana
dashboard. deploy/ adds a Compose setup (PostgreSQL, Prometheus, Grafana,
Jaeger) and a Helm chart; cmd/warp-loadtest loads a server and reports
request and delivery latencies.

The load tester found a long-standing bug, fixed here: on a server where
no account data had been written, every incremental /sync re-sent the push
rules and returned at once, and an all-zero token was taken for an initial
sync, so long polls became busy loops.

The PostgreSQL migrations and search dialect compile but were not run
against a live PostgreSQL in this change.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

server/internal/storage/sqlstore/push.go

@@ -143,8 +143,8 @@ func (s *Push) DeletePushersForDevices(ctx context.Context, localpart string, de
 // to NUMERIC, which does not scan into an int64.
 const (
 	insertNotificationSQL = `INSERT INTO push_notifications
-		(id, user_id, room_id, event_id, event_pos, ts_ms, highlight, actions, is_read)
-		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
+		(id, user_id, room_id, event_id, event_pos, thread_id, ts_ms, highlight, actions, is_read)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
 
 	bumpRoomCountsSQL = `INSERT INTO push_room_counts
 		(user_id, room_id, notification_count, highlight_count, stream_pos)
@@ -157,6 +157,10 @@ const (
 	markReadSQL = `UPDATE push_notifications SET is_read = ?
 		WHERE user_id = ? AND room_id = ? AND is_read = ? AND event_pos <= ?`
 
+	// markThreadReadSQL is markReadSQL for one thread, or for the main
+	// timeline as thread ''.
+	markThreadReadSQL = markReadSQL + ` AND thread_id = ?`
+
 	countUnreadSQL = `SELECT COUNT(*), CAST(COALESCE(SUM(CASE WHEN highlight THEN 1 ELSE 0 END), 0) AS BIGINT)
 		FROM push_notifications WHERE user_id = ? AND room_id = ? AND is_read = ?`
 
@@ -174,7 +178,7 @@ const (
 	selectUnreadTotalSQL = `SELECT CAST(COALESCE(SUM(notification_count), 0) AS BIGINT)
 		FROM push_room_counts WHERE user_id = ?`
 
-	notificationColumns = `id, user_id, room_id, event_id, event_pos, ts_ms, highlight, actions, is_read`
+	notificationColumns = `id, user_id, room_id, event_id, event_pos, thread_id, ts_ms, highlight, actions, is_read`
 
 	pruneNotificationsSQL = `DELETE FROM push_notifications WHERE is_read = ? AND ts_ms < ?`
 )
@@ -196,7 +200,7 @@ func (s *Push) AddNotifications(ctx context.Context, ns []storage.NewNotificatio
 				return err
 			}
 			if _, err := tx.ExecContext(ctx, s.q(insertNotificationSQL), pos, n.UserID, n.RoomID, n.EventID,
-				n.EventPos, n.TimestampMS, n.Highlight, string(n.Actions), false); err != nil {
+				n.EventPos, n.ThreadID, n.TimestampMS, n.Highlight, string(n.Actions), false); err != nil {
 				return fmt.Errorf("write a notification for %q: %w", n.UserID, err)
 			}
 			highlight := 0
@@ -218,14 +222,19 @@ func (s *Push) AddNotifications(ctx context.Context, ns []storage.NewNotificatio
 // The counts are recounted from the rows that are still unread rather than
 // decremented, so that whatever got them out of step — a crash between
 // two writes of an older version, a row pruned by hand — is put right by
-// the next receipt.
-func (s *Push) MarkNotificationsRead(ctx context.Context, userID, roomID string, upTo int64) (int64, bool, error) {
+// the next receipt. The room's counts stay the room's total across every
+// thread whichever thread the receipt read.
+func (s *Push) MarkNotificationsRead(ctx context.Context, userID, roomID, thread string, upTo int64) (int64, bool, error) {
 	var (
 		pos   int64
 		moved bool
 	)
 	err := s.tx(ctx, func(tx *sql.Tx) error {
-		res, err := tx.ExecContext(ctx, s.q(markReadSQL), true, userID, roomID, false, upTo)
+		query, args := markReadSQL, []any{true, userID, roomID, false, upTo}
+		if thread != storage.AllThreads {
+			query, args = markThreadReadSQL, append(args, thread)
+		}
+		res, err := tx.ExecContext(ctx, s.q(query), args...)
 		if err != nil {
 			return fmt.Errorf("mark %q's notifications read: %w", userID, err)
 		}
@@ -291,6 +300,53 @@ func (s *Push) NotificationCounts(ctx context.Context, userID string, roomIDs []
 	return out, nil
 }
 
+// ThreadNotificationCounts implements storage.PushStore.
+//
+// These are counted from the unread rows rather than kept like the room
+// totals: a room has few unread notifications at a time, they are a range
+// of an index, and a table of per-thread counts would be one more thing
+// every write and every receipt has to keep in step.
+func (s *Push) ThreadNotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]map[string]storage.NotificationCounts, error) {
+	out := make(map[string]map[string]storage.NotificationCounts, len(roomIDs))
+	if len(roomIDs) == 0 {
+		return out, nil
+	}
+
+	query := `SELECT room_id, thread_id, COUNT(*),
+			CAST(COALESCE(SUM(CASE WHEN highlight THEN 1 ELSE 0 END), 0) AS BIGINT)
+		FROM push_notifications
+		WHERE user_id = ? AND is_read = ? AND room_id IN (` + placeholders(len(roomIDs)) + `)
+		GROUP BY room_id, thread_id`
+	args := make([]any, 0, len(roomIDs)+2)
+	args = append(args, userID, false)
+	for _, id := range roomIDs {
+		args = append(args, id)
+	}
+	rows, err := s.db.QueryContext(ctx, s.q(query), args...)
+	if err != nil {
+		return nil, fmt.Errorf("read %q's thread notification counts: %w", userID, err)
+	}
+	defer func() { _ = rows.Close() }()
+	for rows.Next() {
+		var roomID, thread string
+		var notifications, highlights int64
+		if err := rows.Scan(&roomID, &thread, &notifications, &highlights); err != nil {
+			return nil, fmt.Errorf("read %q's thread notification counts: %w", userID, err)
+		}
+		if out[roomID] == nil {
+			out[roomID] = map[string]storage.NotificationCounts{}
+		}
+		out[roomID][thread] = storage.NotificationCounts{
+			Notifications: int(notifications),
+			Highlights:    int(highlights),
+		}
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("read %q's thread notification counts: %w", userID, err)
+	}
+	return out, nil
+}
+
 // NotificationCountChanges implements storage.PushStore.
 func (s *Push) NotificationCountChanges(ctx context.Context, userID string, from, to int64) ([]string, error) {
 	rows, err := s.db.QueryContext(ctx, s.q(selectCountChangesSQL), userID, from, to)
@@ -345,7 +401,7 @@ func (s *Push) Notifications(ctx context.Context, userID string, before int64, l
 	for rows.Next() {
 		var n storage.Notification
 		var actions string
-		if err := rows.Scan(&n.ID, &n.UserID, &n.RoomID, &n.EventID, &n.EventPos, &n.TimestampMS,
+		if err := rows.Scan(&n.ID, &n.UserID, &n.RoomID, &n.EventID, &n.EventPos, &n.ThreadID, &n.TimestampMS,
 			&n.Highlight, &actions, &n.Read); err != nil {
 			return nil, fmt.Errorf("read %q's notifications: %w", userID, err)
 		}

server/internal/storage/sqlstore/relations.go

@@ -0,0 +1,495 @@
+package sqlstore
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+
+	"github.com/dariuszjaniak/warp/server/internal/storage"
+)
+
+// The relation and search half of the room store: the reads behind
+// /relations, /threads, the bundled summaries in unsigned.m.relations, and
+// /search.
+//
+// Relations need nothing but the two columns on roomserver_events and their
+// index, and are written once for both engines like everything else here.
+// Search is the one place the engines really differ — SQLite has FTS5,
+// PostgreSQL has tsvector — so the statements that touch the index come
+// from the driver as a SearchDialect, and only the shape of the query is
+// shared.
+
+// SearchDialect is one engine's full-text index.
+//
+// Every statement is written against the table roomserver_event_search,
+// referred to as Name where it is read, with `?` placeholders that are
+// rebound like any other statement.
+type SearchDialect struct {
+	// Insert adds one event, taking (stream_ordering, room_id, event_id,
+	// search_key, sender, event_type, body) in that order.
+	Insert string
+
+	// DeleteEvent removes an event's row, taking (room_id, event_id).
+	DeleteEvent string
+
+	// DeleteRoom removes a room's rows, taking (room_id).
+	DeleteRoom string
+
+	// From is the FROM clause the search reads, and Name what its columns
+	// are qualified with. SQLite's FTS5 cannot be aliased — MATCH and
+	// bm25() take the table's own name — so the two are separate.
+	From string
+	Name string
+
+	// Position is the column holding a row's stream ordering.
+	Position string
+
+	// Match is the condition selecting the rows that match. It takes the
+	// engine query once.
+	Match string
+
+	// Rank is the relevance of a matching row, higher being better. It
+	// takes the engine query RankArgs times.
+	Rank     string
+	RankArgs int
+
+	// Query turns what a user typed into the engine's query syntax. It
+	// returns "" for a term with nothing searchable in it, which matches
+	// nothing.
+	Query func(term string) string
+}
+
+// WithSearch sets the full-text index the store writes and reads.
+func (s *Rooms) WithSearch(d *SearchDialect) *Rooms {
+	s.search = d
+	return s
+}
+
+// indexEvent adds an event that has just joined a timeline to the search
+// index, if it has anything to find it by.
+func (s *Rooms) indexEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEvent, ordering int64) error {
+	if s.search == nil || e.SearchKey == "" || strings.TrimSpace(e.SearchText) == "" {
+		return nil
+	}
+	_, err := tx.ExecContext(ctx, s.q(s.search.Insert),
+		ordering, e.RoomID, e.EventID, e.SearchKey, e.Sender, e.Type, e.SearchText)
+	if err != nil {
+		return fmt.Errorf("index event %s for search: %w", e.EventID, err)
+	}
+	return nil
+}
+
+// unindexEvent removes an event from the search index.
+func (s *Rooms) unindexEvent(ctx context.Context, tx *sql.Tx, roomID, eventID string) error {
+	if s.search == nil {
+		return nil
+	}
+	if _, err := tx.ExecContext(ctx, s.q(s.search.DeleteEvent), roomID, eventID); err != nil {
+		return fmt.Errorf("remove event %s from the search index: %w", eventID, err)
+	}
+	return nil
+}
+
+// Relations implements storage.RoomStore.
+func (s *Rooms) Relations(ctx context.Context, req storage.RelationsRequest) ([]storage.StoredEvent, error) {
+	if req.Limit <= 0 || len(req.ParentIDs) == 0 {
+		return nil, nil
+	}
+
+	conditions := []string{
+		`e.room_id = ?`,
+		`e.relates_to_id IN (` + placeholders(len(req.ParentIDs)) + `)`,
+		`NOT e.outlier`,
+	}
+	args := []any{req.RoomID}
+	for _, id := range req.ParentIDs {
+		args = append(args, id)
+	}
+	if req.RelType != "" {
+		conditions = append(conditions, `e.rel_type = ?`)
+		args = append(args, req.RelType)
+	} else {
+		// A parent's own row has relates_to_id '' and can never match an
+		// IN list of event IDs, but '' itself could be passed; it names no
+		// event and has no children.
+		conditions = append(conditions, `e.rel_type <> ''`)
+	}
+	if req.EventType != "" {
+		conditions = append(conditions, `e.event_type = ?`)
+		args = append(args, req.EventType)
+	}
+	if req.Low != 0 {
+		conditions = append(conditions, `e.stream_ordering > ?`)
+		args = append(args, req.Low)
+	}
+	if req.High != 0 {
+		conditions = append(conditions, `e.stream_ordering <= ?`)
+		args = append(args, req.High)
+	}
+	order := `ASC`
+	if req.Direction == storage.DirectionBackward {
+		order = `DESC`
+	}
+	args = append(args, req.Limit)
+
+	query := `SELECT ` + eventColumns + `
+		FROM roomserver_events e
+		WHERE ` + strings.Join(conditions, " AND ") + `
+		ORDER BY e.stream_ordering ` + order + `
+		LIMIT ?`
+	rows, err := s.db.QueryContext(ctx, s.q(query), args...)
+	if err != nil {
+		return nil, fmt.Errorf("read the relations of %v in %q: %w", req.ParentIDs, req.RoomID, err)
+	}
+	events, err := scanEvents(rows)
+	if err != nil {
+		return nil, fmt.Errorf("read the relations of %v in %q: %w", req.ParentIDs, req.RoomID, err)
+	}
+	return events, nil
+}
+
+// ChildEventIDs implements storage.RoomStore.
+func (s *Rooms) ChildEventIDs(ctx context.Context, roomID string, parentIDs []string, relType, eventType string) ([]string, error) {
+	if len(parentIDs) == 0 {
+		return nil, nil
+	}
+	query := `SELECT event_id FROM roomserver_events
+		WHERE room_id = ? AND relates_to_id IN (` + placeholders(len(parentIDs)) + `)
+		AND rel_type <> '' AND NOT outlier`
+	args := []any{roomID}
+	for _, id := range parentIDs {
+		args = append(args, id)
+	}
+	if relType != "" {
+		query += ` AND rel_type = ?`
+		args = append(args, relType)
+	}
+	if eventType != "" {
+		query += ` AND event_type = ?`
+		args = append(args, eventType)
+	}
+	query += ` ORDER BY stream_ordering`
+
+	ids, err := s.queryStrings(ctx, query, args...)
+	if err != nil {
+		return nil, fmt.Errorf("read the children of %v in %q: %w", parentIDs, roomID, err)
+	}
+	return ids, nil
+}
+
+// RelationAggregates implements storage.RoomStore.
+//
+// It groups by parent, relation type and sender, and joins each group's
+// newest position back onto the events table for its event ID: a portable
+// spelling of "the row with the largest stream ordering in each group",
+// which neither engine can say in one aggregate.
+func (s *Rooms) RelationAggregates(ctx context.Context, roomID string, parentIDs []string, high int64) ([]storage.RelationAggregate, error) {
+	if len(parentIDs) == 0 {
+		return nil, nil
+	}
+	inner := `SELECT relates_to_id, rel_type, sender, COUNT(*) AS n, MAX(stream_ordering) AS latest
+		FROM roomserver_events
+		WHERE room_id = ? AND relates_to_id IN (` + placeholders(len(parentIDs)) + `)
+		AND rel_type IN (?, ?, ?) AND NOT outlier`
+	args := []any{roomID}
+	for _, id := range parentIDs {
+		args = append(args, id)
+	}
+	args = append(args, storage.RelThread, storage.RelReplace, storage.RelReference)
+	if high != 0 {
+		inner += ` AND stream_ordering <= ?`
+		args = append(args, high)
+	}
+	inner += ` GROUP BY relates_to_id, rel_type, sender`
+
+	query := `SELECT g.relates_to_id, g.rel_type, g.sender, g.n, g.latest, e.event_id
+		FROM (` + inner + `) g
+		JOIN roomserver_events e ON e.stream_ordering = g.latest`
+	rows, err := s.db.QueryContext(ctx, s.q(query), args...)
+	if err != nil {
+		return nil, fmt.Errorf("summarise the relations of %d events in %q: %w", len(parentIDs), roomID, err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	var out []storage.RelationAggregate
+	for rows.Next() {
+		var a storage.RelationAggregate
+		if err := rows.Scan(&a.ParentID, &a.RelType, &a.Sender, &a.Count, &a.LatestOrdering, &a.LatestEventID); err != nil {
+			return nil, fmt.Errorf("summarise the relations in %q: %w", roomID, err)
+		}
+		out = append(out, a)
+	}
+	return out, rows.Err()
+}
+
+// RelationEventIDs implements storage.RoomStore.
+//
+// The limit is per parent in spirit and applied over the whole set: the
+// summary it feeds is a sample, and a parent with more children than the
+// limit is rare enough that one query beats one per parent.
+func (s *Rooms) RelationEventIDs(ctx context.Context, roomID string, parentIDs []string, relType string, high int64, limit int) (map[string][]string, error) {
+	if len(parentIDs) == 0 || limit <= 0 {
+		return nil, nil
+	}
+	query := `SELECT relates_to_id, event_id FROM roomserver_events
+		WHERE room_id = ? AND relates_to_id IN (` + placeholders(len(parentIDs)) + `)
+		AND rel_type = ? AND NOT outlier`
+	args := []any{roomID}
+	for _, id := range parentIDs {
+		args = append(args, id)
+	}
+	args = append(args, relType)
+	if high != 0 {
+		query += ` AND stream_ordering <= ?`
+		args = append(args, high)
+	}
+	query += ` ORDER BY stream_ordering LIMIT ?`
+	args = append(args, limit*len(parentIDs))
+
+	rows, err := s.db.QueryContext(ctx, s.q(query), args...)
+	if err != nil {
+		return nil, fmt.Errorf("read the %s relations in %q: %w", relType, roomID, err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	out := map[string][]string{}
+	for rows.Next() {
+		var parent, id string
+		if err := rows.Scan(&parent, &id); err != nil {
+			return nil, fmt.Errorf("read the %s relations in %q: %w", relType, roomID, err)
+		}
+		if len(out[parent]) < limit {
+			out[parent] = append(out[parent], id)
+		}
+	}
+	return out, rows.Err()
+}
+
+// Threads implements storage.RoomStore.
+//
+// A thread is a root with at least one m.thread child, and is ordered by its
+// newest child. The root has to be an event of the room this server holds
+// in its timeline: a thread whose root is missing — not yet backfilled, or
+// never sent here — cannot be shown as a thread at all.
+func (s *Rooms) Threads(ctx context.Context, req storage.ThreadsRequest) ([]storage.Thread, error) {
+	if req.Limit <= 0 {
+		return nil, nil
+	}
+	inner := `SELECT relates_to_id AS root, MAX(stream_ordering) AS latest
+		FROM roomserver_events
+		WHERE room_id = ? AND rel_type = ? AND NOT outlier`
+	args := []any{req.RoomID, storage.RelThread}
+	if req.High != 0 {
+		inner += ` AND stream_ordering <= ?`
+		args = append(args, req.High)
+	}
+	inner += ` GROUP BY relates_to_id`
+
+	conditions := []string{`r.room_id = ?`, `NOT r.outlier`}
+	args = append(args, req.RoomID)
+	if req.Before != 0 {
+		conditions = append(conditions, `g.latest < ?`)
+		args = append(args, req.Before)
+	}
+	if req.ParticipatedBy != "" {
+		conditions = append(conditions, `(r.sender = ? OR EXISTS (
+			SELECT 1 FROM roomserver_events p
+			WHERE p.room_id = r.room_id AND p.relates_to_id = g.root AND p.rel_type = ? AND p.sender = ?))`)
+		args = append(args, req.ParticipatedBy, storage.RelThread, req.ParticipatedBy)
+	}
+	args = append(args, req.Limit)
+
+	query := `SELECT g.root, g.latest
+		FROM (` + inner + `) g
+		JOIN roomserver_events r ON r.event_id = g.root
+		WHERE ` + strings.Join(conditions, " AND ") + `
+		ORDER BY g.latest DESC
+		LIMIT ?`
+	rows, err := s.db.QueryContext(ctx, s.q(query), args...)
+	if err != nil {
+		return nil, fmt.Errorf("read the threads of %q: %w", req.RoomID, err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	var out []storage.Thread
+	for rows.Next() {
+		var t storage.Thread
+		if err := rows.Scan(&t.RootID, &t.LatestOrdering); err != nil {
+			return nil, fmt.Errorf("read the threads of %q: %w", req.RoomID, err)
+		}
+		out = append(out, t)
+	}
+	return out, rows.Err()
+}
+
+// searchKeys are the fields the index holds.
+var searchKeys = []string{"content.body", "content.name", "content.topic"}
+
+// Search implements storage.RoomStore.
+func (s *Rooms) Search(ctx context.Context, req storage.SearchRequest) ([]storage.SearchResult, int, error) {
+	if s.search == nil || req.Limit <= 0 || len(req.Rooms) == 0 {
+		return nil, 0, nil
+	}
+	term := s.search.Query(req.Term)
+	if term == "" {
+		return nil, 0, nil
+	}
+
+	pos := s.search.Position
+	col := func(name string) string { return s.search.Name + "." + name }
+	conditions := []string{s.search.Match}
+	args := []any{term}
+
+	// One bounded clause per room: what a user may search of a room is a
+	// window of its stream, and the windows differ from room to room.
+	rooms := make([]string, 0, len(req.Rooms))
+	for _, r := range req.Rooms {
+		clause := col(`room_id`) + ` = ?`
+		args = append(args, r.RoomID)
+		if r.Low != 0 {
+			clause += ` AND ` + pos + ` > ?`
+			args = append(args, r.Low)
+		}
+		if r.High != 0 {
+			clause += ` AND ` + pos + ` <= ?`
+			args = append(args, r.High)
+		}
+		rooms = append(rooms, `(`+clause+`)`)
+	}
+	conditions = append(conditions, `(`+strings.Join(rooms, ` OR `)+`)`)
+
+	keys := req.Keys
+	if len(keys) == 0 {
+		keys = searchKeys
+	}
+	conditions = append(conditions, col(`search_key`)+` IN (`+placeholders(len(keys))+`)`)
+	for _, k := range keys {
+		args = append(args, k)
+	}
+
+	if clause, filterArgs := filterConditionsOn(col(`event_type`), col(`sender`), req.Filter); clause != "" {
+		conditions = append(conditions, clause)
+		args = append(args, filterArgs...)
+	}
+	where := strings.Join(conditions, " AND ")
+
+	var total int
+	countQuery := `SELECT COUNT(*) FROM ` + s.search.From + ` WHERE ` + where
+	if err := s.db.QueryRowContext(ctx, s.q(countQuery), args...).Scan(&total); err != nil {
+		return nil, 0, fmt.Errorf("count search results: %w", err)
+	}
+
+	pageWhere := where
+	pageArgs := append([]any{}, args...)
+	if req.Order == storage.SearchByRecent && req.Before != 0 {
+		pageWhere += ` AND ` + pos + ` < ?`
+		pageArgs = append(pageArgs, req.Before)
+	}
+
+	rankArgs := make([]any, 0, s.search.RankArgs)
+	for range s.search.RankArgs {
+		rankArgs = append(rankArgs, term)
+	}
+	order := `rank DESC, ` + pos + ` DESC`
+	if req.Order == storage.SearchByRecent {
+		order = pos + ` DESC`
+	}
+	query := `SELECT ` + pos + `, ` + s.search.Rank + ` AS rank
+		FROM ` + s.search.From + `
+		WHERE ` + pageWhere + `
+		ORDER BY ` + order + `
+		LIMIT ? OFFSET ?`
+	offset := req.Offset
+	if req.Order == storage.SearchByRecent {
+		offset = 0
+	}
+	queryArgs := append(rankArgs, pageArgs...)
+	queryArgs = append(queryArgs, req.Limit, offset)
+
+	hits, err := s.searchHits(ctx, query, queryArgs)
+	if err != nil {
+		return nil, 0, err
+	}
+	if len(hits) == 0 {
+		return nil, total, nil
+	}
+
+	// The events themselves, read by position in one query and put back in
+	// the order the search ranked them.
+	eventArgs := make([]any, 0, len(hits))
+	for _, h := range hits {
+		eventArgs = append(eventArgs, h.pos)
+	}
+	eventRows, err := s.db.QueryContext(ctx, s.q(`SELECT `+eventColumns+`
+		FROM roomserver_events e WHERE e.stream_ordering IN (`+placeholders(len(hits))+`)`), eventArgs...)
+	if err != nil {
+		return nil, 0, fmt.Errorf("read search results: %w", err)
+	}
+	events, err := scanEvents(eventRows)
+	if err != nil {
+		return nil, 0, fmt.Errorf("read search results: %w", err)
+	}
+	byPos := make(map[int64]storage.StoredEvent, len(events))
+	for _, e := range events {
+		byPos[e.StreamOrdering] = e
+	}
+
+	out := make([]storage.SearchResult, 0, len(hits))
+	for _, h := range hits {
+		// An index row whose event has gone — purged under a search in
+		// flight — is skipped rather than reported.
+		if e, ok := byPos[h.pos]; ok {
+			out = append(out, storage.SearchResult{Event: e, Rank: h.rank})
+		}
+	}
+	return out, total, nil
+}
+
+// searchHit is one row of a search page: where the event is, and how well
+// it matched.
+type searchHit struct {
+	pos  int64
+	rank float64
+}
+
+// searchHits runs the page query of a search.
+func (s *Rooms) searchHits(ctx context.Context, query string, args []any) ([]searchHit, error) {
+	rows, err := s.db.QueryContext(ctx, s.q(query), args...)
+	if err != nil {
+		return nil, fmt.Errorf("search: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	var hits []searchHit
+	for rows.Next() {
+		var h searchHit
+		if err := rows.Scan(&h.pos, &h.rank); err != nil {
+			return nil, fmt.Errorf("search: %w", err)
+		}
+		hits = append(hits, h)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("search: %w", err)
+	}
+	return hits, nil
+}
+
+// queryStrings runs a query returning one string column.
+func (s *Rooms) queryStrings(ctx context.Context, query string, args ...any) ([]string, error) {
+	rows, err := s.db.QueryContext(ctx, s.q(query), args...)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = rows.Close() }()
+
+	var out []string
+	for rows.Next() {
+		var v string
+		if err := rows.Scan(&v); err != nil {
+			return nil, err
+		}
+		out = append(out, v)
+	}
+	return out, rows.Err()
+}

server/internal/storage/sqlstore/rooms.go

@@ -25,6 +25,9 @@ type Rooms struct {
 	db       *sql.DB
 	rebind   func(string) string
 	isUnique func(error) bool
+
+	// search is the engine's full-text index, nil for none.
+	search *SearchDialect
 }
 
 // NewRooms builds a room store.
@@ -266,9 +269,10 @@ const (
 	insertEventSQL = `
 		INSERT INTO roomserver_events (
 			event_id, room_id, stream_ordering, event_type, state_key,
-			sender, depth, origin_server_ts, event_json, state_before_id, outlier)
-		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
-	selectOutlierSQL  = `SELECT outlier FROM roomserver_events WHERE event_id = ?`
+			sender, depth, origin_server_ts, event_json, state_before_id, outlier,
+			relates_to_id, rel_type)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+	selectOutlierSQL  = `SELECT outlier, redacted_by FROM roomserver_events WHERE event_id = ?`
 	promoteOutlierSQL = `
 		UPDATE roomserver_events
 		SET stream_ordering = ?, state_before_id = ?, outlier = ?
@@ -284,8 +288,11 @@ const (
 // the event ID says the two copies are the same event, and the stored one
 // may already have been redacted.
 func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEvent, ordering, snapshotID int64, outlier bool) error {
-	var existingOutlier bool
-	err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), e.EventID).Scan(&existingOutlier)
+	var (
+		existingOutlier bool
+		redactedBy      string
+	)
+	err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), e.EventID).Scan(&existingOutlier, &redactedBy)
 	switch {
 	case err == nil:
 		if existingOutlier && !outlier {
@@ -293,6 +300,13 @@ func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEve
 				ordering, snapshotID, false, e.EventID, true); err != nil {
 				return fmt.Errorf("promote outlier: %w", err)
 			}
+			// It joins the timeline, so it joins the search index — unless
+			// the copy already held was redacted, whose text is gone.
+			if redactedBy == "" {
+				if err := s.indexEvent(ctx, tx, e, ordering); err != nil {
+					return err
+				}
+			}
 		}
 		// The same event twice is not a failure worth propagating: an
 		// event ID is a hash of its contents, so the second copy says
@@ -304,13 +318,17 @@ func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEve
 
 	_, err = tx.ExecContext(ctx, s.q(insertEventSQL),
 		e.EventID, e.RoomID, ordering, e.Type, e.StateKey,
-		e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, outlier)
+		e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, outlier,
+		e.RelatesTo, e.RelType)
 	if err != nil {
 		if s.isUnique(err) {
 			return nil
 		}
 		return fmt.Errorf("insert event: %w", err)
 	}
+	if !outlier {
+		return s.indexEvent(ctx, tx, e, ordering)
+	}
 	return nil
 }
 
@@ -403,9 +421,13 @@ func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []stora
 		// which is already in the timeline and left alone.
 		var write []storage.BackfillEvent
 		promote := map[string]bool{}
+		redacted := map[string]bool{}
 		for _, b := range events {
-			var outlier bool
-			err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), b.Event.EventID).Scan(&outlier)
+			var (
+				outlier    bool
+				redactedBy string
+			)
+			err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), b.Event.EventID).Scan(&outlier, &redactedBy)
 			switch {
 			case errors.Is(err, sql.ErrNoRows):
 				write = append(write, b)
@@ -414,6 +436,7 @@ func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []stora
 			case outlier:
 				write = append(write, b)
 				promote[b.Event.EventID] = true
+				redacted[b.Event.EventID] = redactedBy != ""
 			}
 		}
 		if len(write) == 0 {
@@ -439,13 +462,22 @@ func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []stora
 					ordering, snapshotID, false, e.EventID, true); err != nil {
 					return fmt.Errorf("promote outlier %s: %w", e.EventID, err)
 				}
+				if !redacted[e.EventID] {
+					if err := s.indexEvent(ctx, tx, e, ordering); err != nil {
+						return err
+					}
+				}
 				continue
 			}
 			if _, err := tx.ExecContext(ctx, s.q(insertEventSQL),
 				e.EventID, e.RoomID, ordering, e.Type, e.StateKey,
-				e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, false); err != nil {
+				e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, false,
+				e.RelatesTo, e.RelType); err != nil {
 				return fmt.Errorf("insert event %s: %w", e.EventID, err)
 			}
+			if err := s.indexEvent(ctx, tx, e, ordering); err != nil {
+				return err
+			}
 		}
 		return nil
 	})
@@ -456,7 +488,7 @@ func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []stora
 }
 
 const applyRedactionSQL = `
-	UPDATE roomserver_events SET event_json = ?, redacted_by = ?
+	UPDATE roomserver_events SET event_json = ?, redacted_by = ?, relates_to_id = '', rel_type = ''
 	WHERE room_id = ? AND event_id = ? AND redacted_by = ''`
 
 // applyRedaction replaces a stored event with its redacted form.
@@ -470,7 +502,9 @@ func (s *Rooms) applyRedaction(ctx context.Context, tx *sql.Tx, roomID, targetID
 	if err != nil {
 		return fmt.Errorf("apply redaction to %s: %w", targetID, err)
 	}
-	return nil
+	// What a redacted event said is gone, and so is its way of being found
+	// by what it said.
+	return s.unindexEvent(ctx, tx, roomID, targetID)
 }
 
 const (
@@ -574,7 +608,8 @@ func (s *Rooms) recordTransaction(ctx context.Context, tx *sql.Tx, ref storage.T
 // scanEvent expects.
 const eventColumns = `
 	e.room_id, e.event_id, e.event_type, e.state_key, e.sender,
-	e.depth, e.origin_server_ts, e.event_json, e.stream_ordering, e.redacted_by, e.outlier`
+	e.depth, e.origin_server_ts, e.event_json, e.stream_ordering, e.redacted_by, e.outlier,
+	e.relates_to_id, e.rel_type`
 
 // scanEvent reads one row of eventColumns.
 func scanEvent(scan func(...any) error) (storage.StoredEvent, error) {
@@ -584,7 +619,8 @@ func scanEvent(scan func(...any) error) (storage.StoredEvent, error) {
 		body     string
 	)
 	err := scan(&e.RoomID, &e.EventID, &e.Type, &stateKey, &e.Sender,
-		&e.Depth, &e.OriginServerTS, &body, &e.StreamOrdering, &e.RedactedBy, &e.Outlier)
+		&e.Depth, &e.OriginServerTS, &body, &e.StreamOrdering, &e.RedactedBy, &e.Outlier,
+		&e.RelatesTo, &e.RelType)
 	if err != nil {
 		return storage.StoredEvent{}, err
 	}
@@ -1087,6 +1123,13 @@ var purgeStatements = []string{
 // PurgeRoom implements storage.RoomStore.
 func (s *Rooms) PurgeRoom(ctx context.Context, roomID string) error {
 	err := s.tx(ctx, func(tx *sql.Tx) error {
+		// The search index has no foreign key to cascade from: SQLite's is
+		// a virtual table.
+		if s.search != nil {
+			if _, err := tx.ExecContext(ctx, s.q(s.search.DeleteRoom), roomID); err != nil {
+				return err
+			}
+		}
 		for i, stmt := range purgeStatements {
 			result, err := tx.ExecContext(ctx, s.q(stmt), roomID)
 			if err != nil {

server/internal/storage/sqlstore/sqlstore.go

@@ -45,6 +45,10 @@ type Options struct {
 	// without a check-then-insert race.
 	IsUniqueViolation func(error) bool
 
+	// Search is this engine's full-text search. Nil leaves the index
+	// unwritten and /search finding nothing.
+	Search *SearchDialect
+
 	// Logger is used for migration progress.
 	Logger *slog.Logger
 }
@@ -81,7 +85,7 @@ func New(opts Options) (*Store, error) {
 		log:      opts.Logger,
 		settings: NewSettings(opts.DB, opts.Settings),
 		users:    NewUsers(opts.DB, opts.Rebind, opts.IsUniqueViolation),
-		rooms:    NewRooms(opts.DB, opts.Rebind, opts.IsUniqueViolation),
+		rooms:    NewRooms(opts.DB, opts.Rebind, opts.IsUniqueViolation).WithSearch(opts.Search),
 		sync:     NewSync(opts.DB, opts.Rebind, opts.IsUniqueViolation),
 		fed:      NewFederation(opts.DB, opts.Rebind),
 		media:    NewMedia(opts.DB, opts.Rebind, opts.IsUniqueViolation),

server/internal/storage/sqlstore/timeline.go

@@ -183,6 +183,12 @@ func (s *Rooms) StreamEvents(ctx context.Context, after, upTo int64, limit int)
 // not_types win over types for an event matching both — the behaviour the
 // spec requires.
 func filterConditions(f storage.TimelineFilter) (string, []any) {
+	return filterConditionsOn(`e.event_type`, `e.sender`, f)
+}
+
+// filterConditionsOn is filterConditions against named columns, for a
+// table other than roomserver_events.
+func filterConditionsOn(typeColumn, senderColumn string, f storage.TimelineFilter) (string, []any) {
 	var (
 		clauses []string
 		args    []any
@@ -198,10 +204,10 @@ func filterConditions(f storage.TimelineFilter) (string, []any) {
 		clauses = append(clauses, clause)
 		args = append(args, clauseArgs...)
 	}
-	add(`e.event_type`, f.Types, false)
-	add(`e.event_type`, f.NotTypes, true)
-	add(`e.sender`, f.Senders, false)
-	add(`e.sender`, f.NotSenders, true)
+	add(typeColumn, f.Types, false)
+	add(typeColumn, f.NotTypes, true)
+	add(senderColumn, f.Senders, false)
+	add(senderColumn, f.NotSenders, true)
 
 	if len(clauses) == 0 {
 		return "", nil
@@ -299,6 +305,48 @@ func (s *Rooms) RoomsWithEvents(ctx context.Context, roomIDs []string, from, to
 	return out, rows.Err()
 }
 
+// LatestEventPositions implements storage.RoomStore.
+func (s *Rooms) LatestEventPositions(ctx context.Context, roomIDs, types []string, upTo int64) (map[string]int64, error) {
+	if len(roomIDs) == 0 || len(types) == 0 {
+		return map[string]int64{}, nil
+	}
+	query := `
+		SELECT room_id, MAX(stream_ordering) FROM roomserver_events
+		WHERE room_id IN (` + placeholders(len(roomIDs)) + `)
+		AND event_type IN (` + placeholders(len(types)) + `) AND NOT outlier`
+	args := make([]any, 0, len(roomIDs)+len(types)+1)
+	for _, id := range roomIDs {
+		args = append(args, id)
+	}
+	for _, t := range types {
+		args = append(args, t)
+	}
+	if upTo != 0 {
+		query += ` AND stream_ordering <= ?`
+		args = append(args, upTo)
+	}
+	query += ` GROUP BY room_id`
+
+	rows, err := s.db.QueryContext(ctx, s.q(query), args...)
+	if err != nil {
+		return nil, fmt.Errorf("find the latest activity of %d rooms: %w", len(roomIDs), err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	out := make(map[string]int64, len(roomIDs))
+	for rows.Next() {
+		var (
+			roomID string
+			pos    int64
+		)
+		if err := rows.Scan(&roomID, &pos); err != nil {
+			return nil, fmt.Errorf("find the latest activity of %d rooms: %w", len(roomIDs), err)
+		}
+		out[roomID] = pos
+	}
+	return out, rows.Err()
+}
+
 const selectEventAtStreamSQL = `
 	SELECT e.event_id, e.event_type, e.state_key, e.state_before_id
 	FROM roomserver_events e