Phase 10: hardening and ops

Implements PLAN.md Phase 10.

Simplified Sliding Sync (MSC4186) is served at the unstable
org.matrix.simplified_msc3575 endpoint and advertised in /versions. Lists
are sorted by activity and filtered by DM, invite, encryption, room type,
space and tag; room subscriptions, required_state with wildcards, $ME and
$LAZY, and the to_device, e2ee, account_data, receipts and typing
extensions are supported. Per-connection state is kept in memory and an
unknown pos answers M_UNKNOWN_POS.

Threads and relations: m.relates_to is indexed on roomserver_events
(migration 0012), /relations (with recursion) and /threads are served, and
every event a timeline read returns carries its m.thread, m.replace and
m.reference summaries in unsigned.m.relations. Notifications carry a
thread ID (migration 0013): threaded receipts clear only their thread, and
sync reports unread_thread_notifications when the filter asks.

Spaces: /hierarchy walks a space breadth first, asking other servers about
rooms this one is not in through the new federation hierarchy endpoint;
room_summary describes one room. /search runs on FTS5 (sqlite, pocketbase)
or tsvector (postgres), bounded to what each user may see, with context,
profiles, state, grouping and paging.

The PocketBase driver is real: the sqlite schema and store on an embedded
PocketBase app's database, with media in its filesystem, local or S3
(database.pocketbase.s3). It runs the full storage test suite.

Ops: OpenTelemetry tracing over OTLP/HTTP (tracing.*), with server spans
per route, component spans, and trace propagation on federation, push and
application service calls. New metrics for persisted events, waiting
syncs, sliding sync connections and the database pool, with a Grafana
dashboard. deploy/ adds a Compose setup (PostgreSQL, Prometheus, Grafana,
Jaeger) and a Helm chart; cmd/warp-loadtest loads a server and reports
request and delivery latencies.

The load tester found a long-standing bug, fixed here: on a server where
no account data had been written, every incremental /sync re-sent the push
rules and returned at once, and an all-zero token was taken for an initial
sync, so long polls became busy loops.

The PostgreSQL migrations and search dialect compile but were not run
against a live PostgreSQL in this change.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

server/internal/syncapi/sync.go

@@ -7,9 +7,12 @@ import (
 	"slices"
 	"time"
 
+	"go.opentelemetry.io/otel/attribute"
+
 	"github.com/dariuszjaniak/warp/server/internal/matrix/event"
 	"github.com/dariuszjaniak/warp/server/internal/roomserver"
 	"github.com/dariuszjaniak/warp/server/internal/storage"
+	"github.com/dariuszjaniak/warp/server/internal/tracing"
 )
 
 // GET /sync: the endpoint a client spends its life in.
@@ -103,6 +106,12 @@ type JoinedRoom struct {
 	// it out rather than claiming zero: an absent count says nothing,
 	// where a zero says "you have read everything".
 	UnreadNotifications *UnreadNotifications `json:"unread_notifications,omitempty"`
+
+	// UnreadThreadNotifications is the counts of each thread with
+	// something unread, keyed by the thread root's event ID, for a filter
+	// that asks for them; UnreadNotifications then counts the main
+	// timeline alone.
+	UnreadThreadNotifications map[string]UnreadNotifications `json:"unread_thread_notifications,omitempty"`
 }
 
 // InvitedRoom is a room the user has been invited to. It carries no
@@ -266,7 +275,10 @@ func (s *API) Sync(ctx context.Context, req SyncRequest) (Response, error) {
 // blocking on a change it has just missed. The listener is returned rather
 // than closed here because the caller is the one that knows whether it is
 // about to wait on it.
-func (s *API) attempt(ctx context.Context, req SyncRequest, filter Filter, since Token) (Response, bool, *Listener, error) {
+func (s *API) attempt(ctx context.Context, req SyncRequest, filter Filter, since Token) (_ Response, _ bool, _ *Listener, err error) {
+	ctx, span := tracing.Start(ctx, "syncapi.sync", attribute.Bool("matrix.initial", since.IsInitial()))
+	defer func() { tracing.End(span, err) }()
+
 	rooms, err := s.rooms.UserMemberships(ctx, req.User, nil)
 	if err != nil {
 		return Response{}, false, nil, err
@@ -333,17 +345,18 @@ func (s *API) build(ctx context.Context, req SyncRequest, filter Filter, since T
 	if err := s.buildRooms(ctx, req, filter, since, to, rooms, &resp); err != nil {
 		return Response{}, false, err
 	}
-	if err := s.unreadNotifications(ctx, req.User, &resp); err != nil {
+	if err := s.unreadNotifications(ctx, req.User, filter.UnreadThreadNotifications(), &resp); err != nil {
 		return Response{}, false, err
 	}
 
-	accountData, err := s.globalAccountData(ctx, req, since.AccountData, to.AccountData, filter)
+	accountData, err := s.globalAccountData(ctx, req, since.AccountData, to.AccountData,
+		req.Since == "" || req.FullState, filter)
 	if err != nil {
 		return Response{}, false, err
 	}
 	resp.AccountData.Events = accountData
 
-	toDevice, toDevicePos, err := s.toDeviceMessages(ctx, req.Localpart, req.Device, since.ToDevice, to.ToDevice)
+	toDevice, toDevicePos, err := s.toDeviceMessages(ctx, req.Localpart, req.Device, since.ToDevice, to.ToDevice, 0)
 	if err != nil {
 		return Response{}, false, err
 	}
@@ -373,7 +386,7 @@ func (s *API) build(ctx context.Context, req SyncRequest, filter Filter, since T
 	}
 
 	resp.NextBatch = next.String()
-	return resp, isInteresting(resp, since), nil
+	return resp, isInteresting(resp, req.Since == ""), nil
 }
 
 // isInteresting reports whether a response is worth returning rather than
@@ -385,8 +398,13 @@ func (s *API) build(ctx context.Context, req SyncRequest, filter Filter, since T
 // into a busy loop. So the test is whether the response has anything in it,
 // with an initial sync as the exception: a client that has just logged in
 // needs an answer whatever is in it.
-func isInteresting(resp Response, since Token) bool {
-	if since.IsInitial() {
+//
+// Initial means the client sent no ?since=, not that the token it sent is
+// all zeroes. On a server whose streams have not moved yet — no events, no
+// receipts, nothing — "s0" is a client's every token, and a sync that took
+// that for a first one would never wait.
+func isInteresting(resp Response, initial bool) bool {
+	if initial {
 		return true
 	}
 	return len(resp.Rooms.Join) > 0 ||
@@ -665,7 +683,8 @@ func (s *API) roomEphemeral(ctx context.Context, userID, roomID string, window r
 }
 
 // globalAccountData reads the account data that is not about a room.
-func (s *API) globalAccountData(ctx context.Context, req SyncRequest, from, to int64, filter Filter) ([]AccountDataEvent, error) {
+// initial is whether this is the client's first read of it.
+func (s *API) globalAccountData(ctx context.Context, req SyncRequest, from, to int64, initial bool, filter Filter) ([]AccountDataEvent, error) {
 	entries, err := s.accountDataEntries(ctx, req.Localpart, from, to)
 	if err != nil {
 		return nil, err
@@ -673,7 +692,13 @@ func (s *API) globalAccountData(ctx context.Context, req SyncRequest, from, to i
 	// Every user has push rules, written or not: a user who has never
 	// changed one has the server defaults, and a client reads its rules
 	// from the account data of its first sync whichever it is.
-	if from == 0 && s.push != nil && !slices.ContainsFunc(entries, func(e storage.AccountDataEntry) bool {
+	//
+	// "First" is the client's first sync, not a token whose account data
+	// position is zero: on a server where nobody has written any account
+	// data yet every token has a zero there, and treating each sync as the
+	// first would send the rules every time — which makes every sync
+	// "interesting", and turns every long poll into a busy loop.
+	if initial && s.push != nil && !slices.ContainsFunc(entries, func(e storage.AccountDataEntry) bool {
 		return e.RoomID == "" && e.DataType == pushRulesType
 	}) {
 		entries = append(entries, storage.AccountDataEntry{DataType: pushRulesType})