diff --git a/Makefile b/Makefile index 37c89ac..15bb822 100644 --- a/Makefile +++ b/Makefile @@ -107,6 +107,19 @@ complement-federation: ## Run the Complement federation subset docker: ## Build the container image docker build -t warp:$(VERSION) . +.PHONY: loadtest +loadtest: ## Load a test server: make loadtest ARGS="-url http://localhost:8008 -users 50" + cd $(SERVER_DIR) && $(GO) run ./cmd/warp-loadtest $(ARGS) + +.PHONY: compose-up +compose-up: ## Run Warp, PostgreSQL and the observability stack with Docker Compose + cd deploy/compose && docker compose --profile observability up -d --build + +.PHONY: helm-lint +helm-lint: ## Lint and render the Helm chart (needs helm) + helm lint deploy/helm/warp + helm template warp deploy/helm/warp > /dev/null + .PHONY: clean clean: ## Remove build output rm -rf $(BIN_DIR) coverage.out coverage.html diff --git a/PLAN.md b/PLAN.md index 915caf4..26b3faf 100644 --- a/PLAN.md +++ b/PLAN.md @@ -407,6 +407,10 @@ the `/_warp/admin/*` API used by the admin panel. 4. **PocketBase scope**: full storage driver as specified, or only media filesystem + admin auth on top of the `sqlite` driver? The latter is far less work for most of the benefit. Decide after the `sqlite` driver is done. + *Decided in Phase 10:* close to the latter. The pocketbase driver runs the + sqlite schema and store on PocketBase's own database connection and puts + media in PocketBase's filesystem (local or S3); PocketBase's admin auth + and realtime are not wired up yet. See docs/deployment.md. 5. **Desktop sync engine**: Go (`mautrix-go`) as planned, or keep `matrix-js-sdk` in the webview for maximum code sharing with web? Prototype both in a week and pick. diff --git a/README.md b/README.md index e78f56f..3c3eca0 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,32 @@ what exists today. ## Status +**Phase 10 — hardening and ops. Complete.** Simplified Sliding Sync +(MSC4186) is served and advertised, so Element X and the matrix-rust-sdk +skip `/sync`: room lists sorted by activity with filters for DMs, invites, +encryption, room types, spaces and tags; room subscriptions; required state +picked tuple by tuple, `$LAZY` members included; and the to-device, E2EE, +account data, receipt and typing extensions, over per-connection state that +answers `M_UNKNOWN_POS` when it has been forgotten. Threads work: +`/relations` (with recursion) and `/threads`, and every event served +carries its thread summary, latest edit and references in +`unsigned.m.relations`. Spaces have `/hierarchy`, walked across servers +through the federation hierarchy endpoint, and `room_summary`. `/search` +runs on SQLite's FTS5 or PostgreSQL's `tsvector`, bounded to what each user +may see, with context, profiles, state and grouping. The PocketBase driver +is real: the SQLite schema inside an embedded PocketBase app, with media in +its filesystem — local or S3 — and it passes the same storage tests as +SQLite. OpenTelemetry tracing follows a request from the client through +federation, push and application service calls; `/metrics` gains event, +sync and database pool metrics, with a Grafana dashboard over them. There +is a Compose setup, a Helm chart and a load tester (`warp-loadtest`), and +the load tester found a long-standing bug fixed here: on a server where no +account data had been written yet, every `/sync` long poll returned at +once. See [docs/deployment.md](docs/deployment.md); the known gap — +sliding sync connection state lost on restart — is in +[docs/sync.md](docs/sync.md). Unread counts are kept per thread for a +client that asks ([docs/push.md](docs/push.md)). + **Phase 9 — application services and the admin API. Complete.** Bridges and bots can be registered from their registration files. A service acts as its own sender or, with `?user_id=`, as any user in its namespace; @@ -176,9 +202,9 @@ counters. See [docs/admin-api.md](docs/admin-api.md). Also in place: argon2id passwords, hashed access and refresh tokens, per-endpoint rate limiting, YAML + environment configuration, structured -logging, graceful shutdown, the storage interfaces with a working SQLite and -PostgreSQL driver (and a PocketBase stub), the embedded migration runner and -the runtime settings service. +logging, graceful shutdown, the storage interfaces with SQLite, PostgreSQL +and PocketBase drivers, the embedded migration runner and the runtime +settings service. Complement, the official integration suite, is wired up for both profiles. `make complement` runs the account, room and stream tests with @@ -246,6 +272,7 @@ security-relevant setting should never be silently ignored. | `warp generate-keys [--output path] [--key-id id]` | create the ed25519 signing key | | `warp create-account --username name [--admin]` | create a local account, straight in the database | | `warp version` | version and compiled-in storage drivers | +| `warp-loadtest [-users n] [-rooms n] [-rate r] [-duration d] [-sliding]` | a separate binary that loads a test server with users, rooms, messages and syncs, and reports latencies; see [docs/deployment.md](docs/deployment.md#load-testing) | `create-account` talks to the database rather than to a running server, so the first account on a new install can be made before anything is listening @@ -302,16 +329,20 @@ Linux. ``` server/ the homeserver (Go module) cmd/warp/ the binary: serve, migrate, generate-keys, create-account, version + cmd/warp-loadtest/ the load tester internal/ config/ YAML + env configuration logging/ metrics/ slog and Prometheus setup + tracing/ OpenTelemetry: server and client spans, trace propagation httputil/ Matrix error envelope, CORS, router, middleware ratelimit/ per-endpoint token buckets federation/ the Server-Server protocol: gate, X-Matrix, discovery, client, keys federationapi/ federation endpoints, the outbound queue, the switchable Module (docs/federation.md) settings/ runtime settings service (the switches) userapi/ accounts, passwords, devices, tokens, account data, deactivation, registration tokens - roomserver/ rooms: the input pipeline (docs/rooms.md) + roomserver/ rooms: the input pipeline (docs/rooms.md), relations and threads (docs/relations.md), spaces (docs/spaces.md), search (docs/search.md) + syncapi/ /sync, sliding sync, /messages, the ephemeral streams (docs/sync.md) + pushrules/ pushserver/ push rules, notification counts, pushers (docs/push.md) mediaapi/ uploads, downloads, thumbnails, remote media (docs/media.md) keyserver/ device keys, one-time keys, cross-signing, key backup (docs/e2ee.md) appservice/ application services: registrations, namespaces, delivery (docs/appservices.md) @@ -334,5 +365,6 @@ server/ the homeserver (Go module) sqlite/ postgres/ pocketbase/ test/complement/ the Complement homeserver image and runner apps/ clients: ui, admin, web, desktop, mobile (Phase 1+) +deploy/ compose/ (with Prometheus, Grafana, Jaeger), helm/warp, grafana/ dashboard docs/ ``` diff --git a/deploy/compose/docker-compose.yml b/deploy/compose/docker-compose.yml new file mode 100644 index 0000000..928e64d --- /dev/null +++ b/deploy/compose/docker-compose.yml @@ -0,0 +1,105 @@ +# Warp with PostgreSQL, and optionally Prometheus, Grafana and Jaeger. +# +# cd deploy/compose +# WARP_SERVER_NAME=matrix.example.org docker compose up -d +# docker compose --profile observability up -d # with the dashboards +# +# The homeserver listens on http://localhost:8008. Put a TLS-terminating +# reverse proxy in front of it before pointing a domain at it; see +# docs/deployment.md. With the observability profile, Grafana is on +# http://localhost:3000 (admin / admin, change it), Prometheus on :9090 and +# the Jaeger UI on :16686. +# +# Create the first administrator once the server is up: +# +# docker compose exec warp warp create-account --username admin --admin + +name: warp + +services: + postgres: + image: postgres:17-alpine + restart: unless-stopped + environment: + POSTGRES_USER: warp + POSTGRES_PASSWORD: ${WARP_POSTGRES_PASSWORD:-warp} + POSTGRES_DB: warp + # The encoding and collation a Matrix server's text wants: C sorts by + # byte, which is what room and event IDs are compared by. + POSTGRES_INITDB_ARGS: --encoding=UTF8 --lc-collate=C --lc-ctype=C + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U warp -d warp"] + interval: 5s + timeout: 5s + retries: 20 + + warp: + build: + context: ../.. + dockerfile: Dockerfile + image: warp:latest + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + # The signing key is created on first start and kept in the volume: + # losing it means other servers no longer recognise this one. + entrypoint: ["/bin/sh", "-c"] + command: + - test -f /data/signing.key || warp generate-keys --output /data/signing.key; exec warp serve + environment: + WARP_SERVER_NAME: ${WARP_SERVER_NAME:-localhost} + WARP_WELL_KNOWN_CLIENT_BASE_URL: ${WARP_PUBLIC_BASEURL:-http://localhost:8008} + WARP_DB_DRIVER: postgres + WARP_DB_CONNECTION_STRING: postgres://warp:${WARP_POSTGRES_PASSWORD:-warp}@postgres:5432/warp?sslmode=disable + WARP_REGISTRATION_ENABLED: ${WARP_REGISTRATION_ENABLED:-false} + WARP_FEDERATION_ENABLED: ${WARP_FEDERATION_ENABLED:-false} + WARP_TRACING_ENABLED: ${WARP_TRACING_ENABLED:-false} + WARP_TRACING_ENDPOINT: http://jaeger:4318 + volumes: + - warp-data:/data + ports: + - "8008:8008" + + prometheus: + image: prom/prometheus:v3.5.0 + profiles: [observability] + restart: unless-stopped + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.retention.time=15d + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + ports: + - "9090:9090" + + grafana: + image: grafana/grafana:12.1.0 + profiles: [observability] + restart: unless-stopped + environment: + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} + GF_AUTH_ANONYMOUS_ENABLED: "false" + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ../grafana:/var/lib/grafana/dashboards/warp:ro + - grafana-data:/var/lib/grafana + ports: + - "3000:3000" + + # Turn on WARP_TRACING_ENABLED=true to send traces here. + jaeger: + image: jaegertracing/jaeger:2.8.0 + profiles: [observability] + restart: unless-stopped + ports: + - "16686:16686" + +volumes: + postgres-data: + warp-data: + prometheus-data: + grafana-data: diff --git a/deploy/compose/grafana/provisioning/dashboards/warp.yml b/deploy/compose/grafana/provisioning/dashboards/warp.yml new file mode 100644 index 0000000..23ea2cc --- /dev/null +++ b/deploy/compose/grafana/provisioning/dashboards/warp.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +# The dashboards in deploy/grafana, mounted read-only into the container. +providers: + - name: warp + folder: Warp + type: file + disableDeletion: true + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards/warp diff --git a/deploy/compose/grafana/provisioning/datasources/prometheus.yml b/deploy/compose/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..6274a6f --- /dev/null +++ b/deploy/compose/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,14 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + - name: Jaeger + type: jaeger + uid: jaeger + access: proxy + url: http://jaeger:16686 diff --git a/deploy/compose/prometheus.yml b/deploy/compose/prometheus.yml new file mode 100644 index 0000000..8125f7b --- /dev/null +++ b/deploy/compose/prometheus.yml @@ -0,0 +1,10 @@ +# Prometheus for the compose setup: scrapes Warp's /metrics. +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: warp + metrics_path: /metrics + static_configs: + - targets: ["warp:8008"] diff --git a/deploy/grafana/warp.json b/deploy/grafana/warp.json new file mode 100644 index 0000000..42773ea --- /dev/null +++ b/deploy/grafana/warp.json @@ -0,0 +1,1139 @@ +{ + "__inputs": [], + "title": "Warp homeserver", + "uid": "warp-homeserver", + "description": "Warp: HTTP, rooms, sync, database and runtime. Built on the metrics /metrics exports.", + "tags": [ + "warp", + "matrix" + ], + "timezone": "browser", + "editable": true, + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "label": "Data source", + "current": {}, + "hide": 0 + }, + { + "name": "job", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "label": "Job", + "query": { + "query": "label_values(warp_build_info, job)", + "refId": "job" + }, + "definition": "label_values(warp_build_info, job)", + "includeAll": true, + "allValue": ".*", + "multi": true, + "refresh": 2, + "current": {} + }, + { + "name": "instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "label": "Instance", + "query": { + "query": "label_values(warp_build_info{job=~\"$job\"}, instance)", + "refId": "instance" + }, + "definition": "label_values(warp_build_info{job=~\"$job\"}, instance)", + "includeAll": true, + "allValue": ".*", + "multi": true, + "refresh": 2, + "current": {} + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Overview", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Version", + "description": "The Warp version each instance reports.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 1, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max by (version) (warp_build_info{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "{{version}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "name" + } + }, + { + "id": 3, + "type": "stat", + "title": "Requests / s", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 4, + "y": 1, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(warp_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 4, + "type": "stat", + "title": "5xx ratio", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 8, + "y": 1, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(warp_http_requests_total{job=~\"$job\", instance=~\"$instance\", status=~\"5..\"}[$__rate_interval])) / sum(rate(warp_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 5, + "type": "stat", + "title": "Events / min", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 1, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(warp_events_persisted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) * 60", + "legendFormat": "", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 6, + "type": "stat", + "title": "Waiting syncs", + "description": "Clients blocked in a /sync or sliding sync long poll: roughly the connected clients.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 16, + "y": 1, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(warp_sync_waiting{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 7, + "type": "stat", + "title": "Federation", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 20, + "y": 1, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max(warp_federation_enabled{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "off", + "color": "text" + }, + "1": { + "text": "on", + "color": "green" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 8, + "type": "row", + "title": "HTTP", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 5, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 9, + "type": "timeseries", + "title": "Requests by route", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 6, + "w": 12, + "h": 9 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(10, sum by (route) (rate(warp_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "{{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Responses by status", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 6, + "w": 12, + "h": 9 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (status) (rate(warp_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 40, + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 11, + "type": "timeseries", + "title": "p95 latency by route (long polls excluded)", + "description": "Sync endpoints block for up to their timeout by design, so they are left out.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 15, + "w": 12, + "h": 9 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(10, histogram_quantile(0.95, sum by (le, route) (rate(warp_http_request_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\", route!~\".*(/sync|simplified_msc3575).*\"}[$__rate_interval]))))", + "legendFormat": "{{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Latency quantiles, all routes but sync", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 15, + "w": 12, + "h": 9 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(warp_http_request_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\", route!~\".*(/sync|simplified_msc3575).*\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(warp_http_request_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\", route!~\".*(/sync|simplified_msc3575).*\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(warp_http_request_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\", route!~\".*(/sync|simplified_msc3575).*\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 13, + "type": "timeseries", + "title": "In-flight requests", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (instance) (warp_http_requests_in_flight{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Federation requests", + "description": "Requests from other servers. Empty while federation is off.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (route) (rate(warp_http_requests_total{job=~\"$job\", instance=~\"$instance\", route=~\".*/_matrix/(federation|key)/.*\"}[$__rate_interval]))", + "legendFormat": "{{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 15, + "type": "row", + "title": "Rooms and sync", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 32, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 16, + "type": "timeseries", + "title": "Events persisted", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 33, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (origin, kind) (rate(warp_events_persisted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{origin}} {{kind}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 40, + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 17, + "type": "timeseries", + "title": "Sync connections", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 33, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(warp_sync_waiting{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "waiting", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(warp_sync_sliding_connections{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "sliding sync connections", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 18, + "type": "row", + "title": "Database", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 41, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 19, + "type": "timeseries", + "title": "Connections", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 42, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (db_name) (go_sql_open_connections{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "open {{db_name}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (db_name) (go_sql_in_use_connections{job=~\"$job\", instance=~\"$instance\"})", + "legendFormat": "in use {{db_name}}", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 20, + "type": "timeseries", + "title": "Waiting for a connection", + "description": "Seconds per second spent waiting for a pooled connection. SQLite and PocketBase have one writer, so some waiting under load is expected.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 42, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (db_name) (rate(go_sql_wait_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{db_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 21, + "type": "row", + "title": "Go runtime", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 50, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 22, + "type": "timeseries", + "title": "Goroutines", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 51, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "go_goroutines{job=~\"$job\", instance=~\"$instance\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 23, + "type": "timeseries", + "title": "Heap in use", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 8, + "y": 51, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 24, + "type": "timeseries", + "title": "CPU", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 16, + "y": 51, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +} diff --git a/deploy/helm/warp/.helmignore b/deploy/helm/warp/.helmignore new file mode 100644 index 0000000..4b15293 --- /dev/null +++ b/deploy/helm/warp/.helmignore @@ -0,0 +1,4 @@ +.git/ +*.swp +*.bak +*.tmp diff --git a/deploy/helm/warp/Chart.yaml b/deploy/helm/warp/Chart.yaml new file mode 100644 index 0000000..1573fb8 --- /dev/null +++ b/deploy/helm/warp/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: warp +description: Warp, a Matrix homeserver in Go — one process, one container. +type: application +# The chart's own version; bump it with every chart change. +version: 0.1.0 +# The Warp version the chart deploys by default (the image tag). +appVersion: "0.10.0" +keywords: [matrix, homeserver, chat] diff --git a/deploy/helm/warp/templates/NOTES.txt b/deploy/helm/warp/templates/NOTES.txt new file mode 100644 index 0000000..4382d9f --- /dev/null +++ b/deploy/helm/warp/templates/NOTES.txt @@ -0,0 +1,17 @@ +Warp is starting as {{ .Values.serverName }}. + +Create the first administrator: + + kubectl exec -n {{ .Release.Namespace }} deploy/{{ include "warp.fullname" . }} -- \ + warp create-account --username admin --admin + +{{- if not .Values.ingress.enabled }} + +Reach it locally: + + kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "warp.fullname" . }} 8008:{{ .Values.service.port }} + curl http://localhost:8008/_matrix/client/versions +{{- end }} + +The signing key {{ if .Values.signingKey.existingSecret }}comes from the secret {{ .Values.signingKey.existingSecret }}{{ else }}is on the data volume, which is kept when the release is deleted{{ end }}. +Losing it means other servers no longer recognise this one. diff --git a/deploy/helm/warp/templates/_helpers.tpl b/deploy/helm/warp/templates/_helpers.tpl new file mode 100644 index 0000000..af35c15 --- /dev/null +++ b/deploy/helm/warp/templates/_helpers.tpl @@ -0,0 +1,46 @@ +{{/* Names and labels shared by the templates. */}} + +{{- define "warp.name" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "warp.fullname" -}} +{{- if contains .Chart.Name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{- define "warp.labels" -}} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} +{{ include "warp.selectorLabels" . }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{- define "warp.selectorLabels" -}} +app.kubernetes.io/name: {{ include "warp.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* The database connection string, defaulted per driver onto the data volume. */}} +{{- define "warp.connectionString" -}} +{{- if .Values.database.connectionString -}} +{{- .Values.database.connectionString -}} +{{- else if eq .Values.database.driver "sqlite" -}} +file:/data/warp.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_pragma=foreign_keys(1) +{{- else if eq .Values.database.driver "pocketbase" -}} +/data/pb_data +{{- else -}} +{{- fail "database.connectionString or database.existingSecret is required for postgres" -}} +{{- end -}} +{{- end -}} + +{{- define "warp.signingKeyPath" -}} +{{- if .Values.signingKey.existingSecret -}} +/etc/warp/keys/{{ .Values.signingKey.existingSecretKey }} +{{- else -}} +/data/signing.key +{{- end -}} +{{- end -}} diff --git a/deploy/helm/warp/templates/deployment.yaml b/deploy/helm/warp/templates/deployment.yaml new file mode 100644 index 0000000..2cef553 --- /dev/null +++ b/deploy/helm/warp/templates/deployment.yaml @@ -0,0 +1,166 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "warp.fullname" . }} + labels: + {{- include "warp.labels" . | nindent 4 }} +spec: + # One homeserver is one process; see values.yaml. + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "warp.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "warp.selectorLabels" . | nindent 8 }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if not .Values.signingKey.existingSecret }} + initContainers: + # Creates the signing key on first start; a key already on the + # volume is left alone. + - name: signing-key + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/bin/sh", "-c", "test -f /data/signing.key || warp generate-keys --output /data/signing.key"] + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + volumeMounts: + - name: data + mountPath: /data + {{- end }} + containers: + - name: warp + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: ["serve"] + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + ports: + - name: http + containerPort: 8008 + protocol: TCP + env: + - name: WARP_SERVER_NAME + value: {{ required "serverName is required" .Values.serverName | quote }} + - name: WARP_WELL_KNOWN_CLIENT_BASE_URL + value: {{ .Values.publicBaseURL | quote }} + - name: WARP_LISTEN_ADDRESS + value: ":8008" + - name: WARP_PRIVATE_KEY + value: {{ include "warp.signingKeyPath" . | quote }} + - name: WARP_DB_DRIVER + value: {{ .Values.database.driver | quote }} + - name: WARP_DB_CONNECTION_STRING + {{- if .Values.database.existingSecret }} + valueFrom: + secretKeyRef: + name: {{ .Values.database.existingSecret }} + key: {{ .Values.database.existingSecretKey }} + {{- else if eq .Values.database.driver "postgres" }} + valueFrom: + secretKeyRef: + name: {{ include "warp.fullname" . }}-database + key: connection-string + {{- else }} + value: {{ include "warp.connectionString" . | quote }} + {{- end }} + - name: WARP_DB_MAX_OPEN_CONNS + value: {{ .Values.database.maxOpenConns | quote }} + - name: WARP_MEDIA_BASE_PATH + value: /data/media + - name: WARP_REGISTRATION_ENABLED + value: {{ .Values.registration.enabled | quote }} + - name: WARP_FEDERATION_ENABLED + value: {{ .Values.federation.enabled | quote }} + - name: WARP_METRICS_ENABLED + value: {{ .Values.metrics.enabled | quote }} + - name: WARP_TRACING_ENABLED + value: {{ .Values.tracing.enabled | quote }} + {{- with .Values.tracing.endpoint }} + - name: WARP_TRACING_ENDPOINT + value: {{ . | quote }} + {{- end }} + - name: WARP_TRACING_SAMPLE_RATIO + value: {{ .Values.tracing.sampleRatio | quote }} + - name: WARP_LOG_FORMAT + value: json + {{- with .Values.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + startupProbe: + httpGet: + path: /healthz + port: http + # Migrations run before the server listens; a large database + # takes a while on the first start after an upgrade. + failureThreshold: 60 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 30 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 10 + timeoutSeconds: 5 + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: /data + - name: tmp + mountPath: /tmp + {{- if .Values.signingKey.existingSecret }} + - name: signing-key + mountPath: /etc/warp/keys + readOnly: true + {{- end }} + volumes: + - name: data + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ .Values.persistence.existingClaim | default (printf "%s-data" (include "warp.fullname" .)) }} + {{- else }} + emptyDir: {} + {{- end }} + - name: tmp + emptyDir: {} + {{- if .Values.signingKey.existingSecret }} + - name: signing-key + secret: + secretName: {{ .Values.signingKey.existingSecret }} + defaultMode: 0400 + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deploy/helm/warp/templates/ingress.yaml b/deploy/helm/warp/templates/ingress.yaml new file mode 100644 index 0000000..f54cb6f --- /dev/null +++ b/deploy/helm/warp/templates/ingress.yaml @@ -0,0 +1,35 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "warp.fullname" . }} + labels: + {{- include "warp.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "warp.fullname" $ }} + port: + name: http + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/helm/warp/templates/pvc.yaml b/deploy/helm/warp/templates/pvc.yaml new file mode 100644 index 0000000..3549a4c --- /dev/null +++ b/deploy/helm/warp/templates/pvc.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "warp.fullname" . }}-data + labels: + {{- include "warp.labels" . | nindent 4 }} + annotations: + # The signing key lives here: deleting the release must not delete it. + helm.sh/resource-policy: keep +spec: + accessModes: + {{- toYaml .Values.persistence.accessModes | nindent 4 }} + {{- with .Values.persistence.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size }} +{{- end }} diff --git a/deploy/helm/warp/templates/secret.yaml b/deploy/helm/warp/templates/secret.yaml new file mode 100644 index 0000000..aff6b3e --- /dev/null +++ b/deploy/helm/warp/templates/secret.yaml @@ -0,0 +1,11 @@ +{{- if and (not .Values.database.existingSecret) (eq .Values.database.driver "postgres") }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "warp.fullname" . }}-database + labels: + {{- include "warp.labels" . | nindent 4 }} +type: Opaque +stringData: + connection-string: {{ include "warp.connectionString" . | quote }} +{{- end }} diff --git a/deploy/helm/warp/templates/service.yaml b/deploy/helm/warp/templates/service.yaml new file mode 100644 index 0000000..ea6af66 --- /dev/null +++ b/deploy/helm/warp/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "warp.fullname" . }} + labels: + {{- include "warp.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + selector: + {{- include "warp.selectorLabels" . | nindent 4 }} diff --git a/deploy/helm/warp/templates/servicemonitor.yaml b/deploy/helm/warp/templates/servicemonitor.yaml new file mode 100644 index 0000000..7878eb5 --- /dev/null +++ b/deploy/helm/warp/templates/servicemonitor.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "warp.fullname" . }} + labels: + {{- include "warp.labels" . | nindent 4 }} + {{- with .Values.metrics.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "warp.selectorLabels" . | nindent 6 }} + endpoints: + - port: http + path: /metrics + interval: {{ .Values.metrics.serviceMonitor.interval }} +{{- end }} diff --git a/deploy/helm/warp/values.yaml b/deploy/helm/warp/values.yaml new file mode 100644 index 0000000..8842fa5 --- /dev/null +++ b/deploy/helm/warp/values.yaml @@ -0,0 +1,134 @@ +# Default values for the Warp chart. +# +# Warp is a monolith: one replica holds the whole homeserver, and a second +# one would be a second homeserver with the same name. The deployment +# therefore runs exactly one pod and replaces it with the Recreate +# strategy, never two at once. + +# There is no published image: build one with `make docker` (or +# `docker build -t /warp: .`) and push it to a registry the +# cluster can pull from. +image: + repository: warp + # Defaults to the chart's appVersion. + tag: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] + +# The Matrix server name: the part after the colon in @user:example.org. +# It cannot be changed once users and rooms exist. +serverName: example.org + +# The URL clients reach the server at, served in /.well-known/matrix/client +# and in login responses. Usually the ingress host over https. +publicBaseURL: https://matrix.example.org + +registration: + enabled: false + +federation: + enabled: false + +database: + # sqlite, postgres or pocketbase. + driver: sqlite + # The connection string. For sqlite and pocketbase the defaults put the + # database on the data volume; for postgres, set it here or, better, + # through existingSecret. + connectionString: "" + # A secret holding the connection string under existingSecretKey, + # instead of connectionString. + existingSecret: "" + existingSecretKey: connection-string + maxOpenConns: 20 + +# The ed25519 signing key. Other servers know this server by it, so it must +# survive the pod: by default it is created on first start on the data +# volume. To manage it yourself, put a key made with `warp generate-keys` +# into a secret and name it here. +signingKey: + existingSecret: "" + existingSecretKey: signing.key + +# The data volume: the signing key, and the sqlite or pocketbase database +# and the media store when those are on local disk. +persistence: + enabled: true + # Use an existing claim instead of creating one. + existingClaim: "" + storageClass: "" + accessModes: [ReadWriteOnce] + size: 20Gi + +metrics: + enabled: true + # A prometheus-operator ServiceMonitor for /metrics. + serviceMonitor: + enabled: false + interval: 30s + labels: {} + +tracing: + enabled: false + # The OTLP/HTTP collector, e.g. http://otel-collector.observability:4318. + endpoint: "" + sampleRatio: 1 + +# Any other WARP_* setting, as plain environment variables. Every field of +# warp.yaml.example has one. +extraEnv: [] +# - name: WARP_LOG_LEVEL +# value: debug + +# Further environment from secrets or config maps. +extraEnvFrom: [] + +service: + type: ClusterIP + port: 8008 + +ingress: + enabled: false + className: "" + annotations: {} + # Long polls hold a request open for up to 30 seconds; a proxy with a + # shorter read timeout cuts them off. For ingress-nginx: + # nginx.ingress.kubernetes.io/proxy-read-timeout: "90" + # nginx.ingress.kubernetes.io/proxy-body-size: "50m" + hosts: + - host: matrix.example.org + paths: + - path: /_matrix + pathType: Prefix + - path: /.well-known/matrix + pathType: Prefix + - path: /_warp + pathType: Prefix + tls: [] + # - secretName: matrix-tls + # hosts: [matrix.example.org] + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 2Gi + +podSecurityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + runAsNonRoot: true + +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + +podAnnotations: {} +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/docs/README.md b/docs/README.md index c194f4b..12b584b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,12 +6,16 @@ | [development.md](development.md) | building, testing, Complement, adding a migration | | [event-core.md](event-core.md) | `internal/matrix`: canonical JSON, events, auth rules, state resolution | | [rooms.md](rooms.md) | `internal/roomserver`: the input pipeline, room storage, known gaps | -| [sync.md](sync.md) | `internal/syncapi`: stream tokens, long polling, filters, the ephemeral streams | +| [sync.md](sync.md) | `internal/syncapi`: stream tokens, long polling, filters, the ephemeral streams, Simplified Sliding Sync | +| [relations.md](relations.md) | threads, edits, reactions and references: `/relations`, `/threads` and `unsigned.m.relations` | +| [spaces.md](spaces.md) | `/hierarchy` across servers, and `room_summary` | +| [search.md](search.md) | `/search` on FTS5 and `tsvector` | | [federation.md](federation.md) | the Server-Server API: discovery, keys, taking in events, joins across servers, the outbound queue, the switch | | [media.md](media.md) | `internal/mediaapi`: uploads, downloads, thumbnails, other servers' media, where the files go | | [e2ee.md](e2ee.md) | `internal/keyserver`: device and one-time keys, cross-signing, device list tracking, key backup | | [push.md](push.md) | `internal/pushrules` and `internal/pushserver`: push rules, notification counts, pushers, the push gateway client | | [appservices.md](appservices.md) | `internal/appservice`: registration files, namespaces, acting as users, delivering events to bridges | | [admin-api.md](admin-api.md) | `/_warp/admin/v1`: the operator API behind the admin panel | +| [deployment.md](deployment.md) | storage drivers, the image, Compose, Helm, metrics and the dashboard, tracing, load testing | The design and roadmap are in [PLAN.md](../PLAN.md) at the repository root. diff --git a/docs/configuration.md b/docs/configuration.md index 479edfc..ccd5f00 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -36,12 +36,20 @@ List-valued variables are comma-separated. A single `-` clears a list: | `listen.federation_address` | `WARP_LISTEN_FEDERATION_ADDRESS` | empty | A second listener serving the same API over TLS, usually `:8448`. Empty = none; see *Federation over TLS*. | | `listen.tls_cert_file` | `WARP_TLS_CERT_FILE` | empty | PEM certificate chain for the federation listener. Required with it. | | `listen.tls_key_file` | `WARP_TLS_KEY_FILE` | empty | PEM key for the federation listener. Required with it. | -| `database.driver` | `WARP_DB_DRIVER` | `sqlite` | `sqlite`, `postgres`, `pocketbase` (not implemented). | -| `database.connection_string` | `WARP_DB_CONNECTION_STRING` | SQLite file in the working directory | | +| `database.driver` | `WARP_DB_DRIVER` | `sqlite` | `sqlite`, `postgres`, `pocketbase`. | +| `database.connection_string` | `WARP_DB_CONNECTION_STRING` | SQLite file in the working directory | A DSN for postgres, a `file:` URI for sqlite, the data directory (PocketBase's `pb_data`) for pocketbase. | | `database.max_open_conns` | `WARP_DB_MAX_OPEN_CONNS` | `10` | Ignored by SQLite: writes are serialised onto one connection. | | `database.max_idle_conns` | `WARP_DB_MAX_IDLE_CONNS` | `5` | | | `database.conn_max_lifetime` | `WARP_DB_CONN_MAX_LIFETIME` | `1h` | | | `database.auto_migrate` | `WARP_DB_AUTO_MIGRATE` | `true` | `false` makes startup fail while migrations are pending. | +| `database.pocketbase.s3.enabled` | `WARP_PB_S3_ENABLED` | `false` | pocketbase only: media content goes to an S3-compatible bucket instead of `pb_data/storage`. | +| `database.pocketbase.s3.bucket` / `.region` / `.endpoint` | `WARP_PB_S3_BUCKET` / `WARP_PB_S3_REGION` / `WARP_PB_S3_ENDPOINT` | empty | All required when S3 is on. | +| `database.pocketbase.s3.access_key` / `.secret` | `WARP_PB_S3_ACCESS_KEY` / `WARP_PB_S3_SECRET` | empty | All required when S3 is on. | +| `database.pocketbase.s3.force_path_style` | `WARP_PB_S3_FORCE_PATH_STYLE` | `false` | For MinIO and other servers without virtual-host buckets. | +| `database.pocketbase.s3.enabled` | `WARP_PB_S3_ENABLED` | `false` | pocketbase only: media content in an S3-compatible bucket instead of `pb_data/storage`. | +| `database.pocketbase.s3.bucket` / `.region` / `.endpoint` | `WARP_PB_S3_BUCKET` / `WARP_PB_S3_REGION` / `WARP_PB_S3_ENDPOINT` | empty | All required with S3 on. | +| `database.pocketbase.s3.access_key` / `.secret` | `WARP_PB_S3_ACCESS_KEY` / `WARP_PB_S3_SECRET` | empty | All required with S3 on. | +| `database.pocketbase.s3.force_path_style` | `WARP_PB_S3_FORCE_PATH_STYLE` | `false` | For MinIO and other servers without virtual-host buckets. | | `client_api.cors_allowed_origins` | `WARP_CORS_ALLOWED_ORIGINS` | `["*"]` | The spec expects any origin. | | `client_api.trust_x_forwarded_for` | `WARP_TRUST_X_FORWARDED_FOR` | `false` | Only behind a proxy that overwrites the header; the rate limiter keys on the result. | | `client_api.access_token_lifetime` | `WARP_ACCESS_TOKEN_LIFETIME` | `0s` | `0` = never expire. With a lifetime, clients may ask for a refresh token at login. | @@ -91,6 +99,10 @@ List-valued variables are comma-separated. A single `-` clears a list: | `logging.add_source` | `WARP_LOG_ADD_SOURCE` | `false` | File and line in every record. | | `metrics.enabled` | `WARP_METRICS_ENABLED` | `true` | | | `metrics.path` | `WARP_METRICS_PATH` | `/metrics` | Served on the same listener. | +| `tracing.enabled` | `WARP_TRACING_ENABLED` | `false` | OpenTelemetry tracing; see [deployment.md](deployment.md#tracing). | +| `tracing.endpoint` | `WARP_TRACING_ENDPOINT` | empty | The OTLP/HTTP collector, e.g. `http://localhost:4318`. Empty falls back to `OTEL_EXPORTER_OTLP_ENDPOINT`, then to `localhost:4318`. | +| `tracing.sample_ratio` | `WARP_TRACING_SAMPLE_RATIO` | `1` | The fraction of new traces kept, 0 to 1. A request that arrives inside a sampled trace is always kept. | +| `tracing.service_name` | `WARP_TRACING_SERVICE_NAME` | `warp` | | The rate limit buckets share one Go type, so per-bucket environment variables would all collide on the same name; they are YAML only. diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..be372c3 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,171 @@ +# Deployment and operations + +What there is for running Warp somewhere other than a laptop: the container +image, a Compose setup, a Helm chart, a Grafana dashboard, tracing, and a +load tester. All of it is under [`deploy/`](../deploy) except the image, +which is the [`Dockerfile`](../Dockerfile) at the root, and the load +tester, which is `server/cmd/warp-loadtest`. + +## Choosing a storage driver + +| Driver | For | Notes | +|---|---|---| +| `postgres` | production | Several connections, so reads and writes run side by side. | +| `sqlite` | development, a single user, a small server | One writer. Everything is one file plus `media.base_path`. | +| `pocketbase` | an "appliance": one binary, one data directory | The sqlite driver's tables inside an embedded [PocketBase](https://pocketbase.io) app's `data.db`, written through PocketBase's own single-writer connection. Media goes into PocketBase's filesystem, which can be an S3 bucket (`database.pocketbase.s3`). | + +The pocketbase driver is the SQLite schema on the SQLite engine, so it is +at parity with sqlite by construction, and the storage tests run against +both. What it adds is PocketBase around it: its system tables in the same +database, its `auxiliary.db` for its own logs, and its filesystem +abstraction for media. Warp does not serve PocketBase's admin UI or API; +the app is reachable from Go through `pocketbase.Database.App()` for +features that want it later. Back up the whole data directory. + +## The container image + +```sh +docker build -t warp . +``` + +One static binary on Alpine, running as UID 10001, with `/data` as its +working directory and volume. The defaults put the SQLite database, the +signing key and media under `/data`. The image does not create a signing +key on its own; the Compose file and the Helm chart do it on first start, +and a plain `docker run` needs one first: + +```sh +docker run --rm -v warp-data:/data warp generate-keys --output /data/signing.key +``` + +## Compose + +[`deploy/compose`](../deploy/compose) runs Warp with PostgreSQL, and with +the `observability` profile, Prometheus, Grafana (with the dashboard +provisioned) and Jaeger: + +```sh +cd deploy/compose +WARP_SERVER_NAME=matrix.example.org docker compose up -d +docker compose --profile observability up -d +docker compose exec warp warp create-account --username admin --admin +``` + +Settings come from the environment, so anything in +[configuration.md](configuration.md) can be added to the `warp` service. +`WARP_TRACING_ENABLED=true` sends traces to the Jaeger container. + +## Helm + +[`deploy/helm/warp`](../deploy/helm/warp) is a chart for one Warp pod. +Warp is one process, so the deployment always runs exactly one replica and +replaces it with the `Recreate` strategy: two pods would be two +homeservers with the same name. + +```sh +helm install warp deploy/helm/warp \ + --set serverName=example.org \ + --set publicBaseURL=https://matrix.example.org \ + --set image.repository=registry.example.org/warp --set image.tag=0.10.0 \ + --set ingress.enabled=true +``` + +There is no published image; build one and push it to a registry the +cluster can pull from. The chart creates a volume for `/data` that is kept +when the release is deleted, because the signing key is on it unless +`signingKey.existingSecret` names a secret holding one. For PostgreSQL, set +`database.driver=postgres` and either `database.connectionString` or +`database.existingSecret`. `metrics.serviceMonitor.enabled` adds a +prometheus-operator `ServiceMonitor`. The ingress needs a read timeout +above the 30-second long poll; `values.yaml` has the ingress-nginx +annotations. + +## Reverse proxy + +Behind a proxy that terminates TLS: + +- Route `/_matrix`, `/.well-known/matrix` and `/_warp` to Warp. +- Keep the proxy's read timeout above `sync.long_poll_timeout` (30 s), or + every idle `/sync` is cut off. +- Allow request bodies of `media.max_upload_bytes` (50 MiB by default). +- Set `client_api.trust_x_forwarded_for: true` only if the proxy + overwrites `X-Forwarded-For`; the rate limiter keys on it. + +## Metrics and the dashboard + +`/metrics` is Prometheus text, on by default. Besides the Go runtime and +process collectors: + +| Metric | | +|---|---| +| `warp_http_requests_total{route,method,status}` | Requests by route pattern (never by path, so room and user IDs cannot explode the cardinality). | +| `warp_http_request_duration_seconds{route,method}` | Latency histogram. Sync endpoints block by design, so leave them out of latency quantiles. | +| `warp_http_requests_in_flight` | | +| `warp_events_persisted_total{origin,kind}` | Room events persisted: `origin` is `local` or `federation`, `kind` is `state` or `message`. | +| `warp_sync_waiting` | Syncs blocked in a long poll, classic and sliding: roughly the connected clients. | +| `warp_sync_sliding_connections` | Sliding sync connections whose state the server holds. | +| `warp_federation_enabled` | 1 while federation is on. | +| `go_sql_*{db_name}` | The connection pool: open, in use, waits. | +| `warp_build_info{version,commit}` | | + +[`deploy/grafana/warp.json`](../deploy/grafana/warp.json) is a Grafana +dashboard over these: overview stats, requests and latency by route, +events, sync connections, the database pool and the Go runtime. Import it +and pick the Prometheus data source; the `job` and `instance` variables +select the server. + +## Tracing + +With `tracing.enabled`, Warp exports OpenTelemetry traces over OTLP/HTTP to +`tracing.endpoint` (or to `OTEL_EXPORTER_OTLP_ENDPOINT`, or to +`localhost:4318`): an OpenTelemetry Collector, Jaeger, Tempo, Honeycomb. + +- Every request is a server span named after its route pattern, with the + method, path, user agent and status. +- Persisting an event, building a sync or a sliding sync response, and + running a search are spans of their own inside it. +- Every request Warp sends — to other servers, to push gateways, to + application services — is a client span, and carries the W3C + `traceparent` on, so a federation request shows up in the same trace as + the client request that caused it. +- A request that arrives with a `traceparent` continues that trace. + +`tracing.sample_ratio` keeps that fraction of new traces; a request inside +a sampled trace is always kept. Off, tracing costs a function call per +span, and the trace context of incoming requests is still passed on. + +## Load testing + +`warp-loadtest` drives a server the way a crowd of clients would: + +```sh +make loadtest ARGS="-users 50 -rooms 10 -rate 0.5 -duration 2m" +# or +go run ./server/cmd/warp-loadtest -url http://localhost:8008 -users 50 -rooms 10 -rate 0.5 -duration 2m -sliding +``` + +It registers the users (answering the dummy and registration-token +stages), creates the rooms and joins everybody to all of them, then for the +length of the run each user sends at `-rate` messages per second while +syncing in a long-poll loop — `/sync`, or sliding sync with `-sliding`. +Each message carries the time it was sent, so the report has, beside the +latency of every kind of request, the **delivery** time: from one user +sending to another user's sync handing it over, which is the number a user +of a chat server feels. + +Point it at a test server with registration open and rate limiting off +(`WARP_RATE_LIMIT_ENABLED=false`), or it measures the rate limiter. It +creates accounts and rooms and never cleans them up. + +A run on a laptop, PocketBase driver, 20 users in 4 rooms at one message a +second each (`-users 20 -rooms 4 -rate 1 -duration 15s`): + +| operation | count | p50 | p99 | +|---|---|---|---| +| send | 280 | 24 ms | 52 ms | +| delivery (`/sync`) | 5320 | 147 ms | 280 ms | +| delivery (sliding sync) | 5320 | 205 ms | 368 ms | + +Every message reached every other user. Those numbers are one writer +serving twenty clients that each re-sync on every message; PostgreSQL's +parallel reads are what moves them. diff --git a/docs/development.md b/docs/development.md index 6d73e9a..6f0ba10 100644 --- a/docs/development.md +++ b/docs/development.md @@ -47,7 +47,8 @@ WARP_SERVER_NAME=dev.local WARP_PRIVATE_KEY=./keys/dev.key ./bin/warp serve 1. Write the same change twice, once per SQL dialect: `server/internal/storage/sqlite/migrations/NNNN_name.sql` and `server/internal/storage/postgres/migrations/NNNN_name.sql`. - Use the next free number; `NNNN` is zero-padded to four digits. + Use the next free number; `NNNN` is zero-padded to four digits. The + pocketbase driver uses the SQLite files as they are. 2. The files are embedded with `embed.FS`, so a rebuild picks them up. 3. `./bin/warp migrate --status` lists what would run, `./bin/warp migrate` applies it. A server started with `database.auto_migrate: true` (the @@ -124,7 +125,9 @@ says why. smaller set that goes through the whole wrapped handler, to catch a route that was written but never registered. - Storage tests run against every driver that is available: - SQLite always, PostgreSQL when `WARP_TEST_POSTGRES_DSN` is set. The + SQLite and PocketBase always, PostgreSQL when `WARP_TEST_POSTGRES_DSN` is + set. PocketBase runs the SQLite migrations on PocketBase's own database, + so a SQLite migration is a PocketBase one too. The PostgreSQL tests share one database, so each group of them truncates the tables it owns first; add a table to `resetRoomTables` or `resetSyncTables` when you add one, or the second test to insert the same diff --git a/docs/media.md b/docs/media.md index 931be0a..c4230c3 100644 --- a/docs/media.md +++ b/docs/media.md @@ -42,9 +42,13 @@ refers to content until it is all there. The blob store is chosen by the storage driver. SQLite and PostgreSQL use the local filesystem under `media.base_path`. A driver that brings its own -file storage implements `storage.BlobProvider`, and PocketBase will — its -filesystem abstraction is local disk or S3 (PLAN.md section 4) — when that -driver is built in Phase 10. +file storage implements `storage.BlobProvider`, and PocketBase does: media +goes into PocketBase's filesystem, under `warp_media/` in +`/storage`, or in an S3-compatible bucket when +`database.pocketbase.s3` is on ([configuration.md](configuration.md)). +PocketBase's filesystem is written a whole file at a time, so an upload is +spooled to `/.warp_tmp` and hashed on the way, then handed +over under its hash; `media.base_path` is not used by that driver. ## Endpoints @@ -235,5 +239,5 @@ with a message saying federation is disabled. `413 M_TOO_LARGE`. See [admin-api.md](admin-api.md). - **No quarantine**, which belongs with the same admin work. - **No animated thumbnails**; `animated=true` gets a still image. -- **No S3** until the PocketBase driver brings its filesystem abstraction; - the `BlobProvider` seam is where it plugs in. +- **S3 only on PocketBase.** The sqlite and postgres drivers keep media on + local disk; S3 comes with the pocketbase driver's filesystem. diff --git a/docs/push.md b/docs/push.md index 83070ad..9024290 100644 --- a/docs/push.md +++ b/docs/push.md @@ -135,7 +135,7 @@ write to the rules, which is what makes the cache safe in one process. | Table (migration `0010_push.sql`) | What | |---|---| -| `push_notifications` | one row per event that notified a user: the event, its position in the room event stream, the actions, highlight, read | +| `push_notifications` | one row per event that notified a user: the event, its position in the room event stream, its thread (`0013_thread_notifications.sql`), the actions, highlight, read | | `push_room_counts` | each user's unread and highlight counts in each room, kept current by the writes, stamped with a notification stream position | | `push_pushers` | pushers | | `push_sequences` | the notification stream's counter | @@ -165,10 +165,46 @@ then recounted from the rows that are still unread rather than decremented, so if anything ever put them out of step, the next receipt puts them right. `/read_markers` goes through the same path. -Notifications are not tracked per thread (threads are Phase 10). So a -receipt counts when it is for the whole room or for its `main` timeline. A -receipt inside a thread changes nothing; thread notifications are cleared -when the main receipt moves past them. +Each notification is filed under the thread its event is in: the root +that its `m.relates_to` names when `rel_type` is `m.thread`, or the main +timeline for anything else (a thread root itself included). A receipt +reads the part of the room its `thread_id` names: + +| `thread_id` | Reads, up to the receipt's event | +|---|---| +| none | the whole room, every thread included | +| `main` | the main timeline only | +| a thread root's event ID | that thread only | + +So reading one thread leaves the main timeline and the other threads +unread, and a client that knows nothing of threads sends unthreaded +receipts and clears the room as it always did. `/read_markers` sends +unthreaded receipts. A local user leaving or being banned reads the whole +room. + +### Per-thread counts + +`push_room_counts` stays the room's total across every thread, which is +what `unread_notifications` reports by default, what sliding sync reports, +and what a push's badge counts. A sync whose filter sets +`room.timeline.unread_thread_notifications` gets the counts split instead: +`unread_notifications` counts the main timeline only, and +`unread_thread_notifications` maps each thread root with something unread +to its `notification_count` and `highlight_count`. A thread with nothing +unread is left out of the map, and a room with no unread threads has no +map, both of which the spec allows. + +The per-thread counts are counted from the unread rows of +`push_notifications` when a sync asks, rather than kept in a table like +the room totals: a room has few unread notifications at a time, they are a +range of an index, and a second table of counts would be one more thing +every write and receipt had to keep in step. They need no stream of their +own either: any change to a thread's counts is a change to its room's, +which stamps the room and brings it down the next sync. + +Notifications stored before migration 0013 are filed by the relation the +roomserver projected onto their events in 0012; one whose event is gone +stays on the main timeline. Read notifications are kept for `push.notification_retention` (30 days by default) for `GET /notifications`, then pruned hourly. Unread ones are kept @@ -227,7 +263,7 @@ to again. | Thing | Why | |---|---| -| `unread_thread_notifications` | needs threads (Phase 10) | +| Per-thread counts in sliding sync and in pushes | sliding sync reports each room's total, and a push's badge is the user's total across rooms; neither has anywhere the spec puts a per-thread count | | Skipping senders in `m.ignored_user_list` | ignored users' events still notify; a client hides them, but they count | | Email pushers | needs a mail server; see above | | Durable evaluation | the emit stage is in memory, so events persisted just before a crash are never evaluated. Their messages are still there; only the count and the push are missed. | diff --git a/docs/relations.md b/docs/relations.md new file mode 100644 index 0000000..b8f1e7d --- /dev/null +++ b/docs/relations.md @@ -0,0 +1,97 @@ +# Relations and threads + +An event can point at another through `content.m.relates_to`: a thread +reply (`m.thread`), an edit (`m.replace`), a reaction (`m.annotation`), a +reference (`m.reference`). None of that is checked by the authorisation +rules. A relation is a client convention, not part of the room's DAG, and a +server can accept an event without understanding what it relates to. What +the server adds is three things: + +1. **The index.** Every event's `m.relates_to` is written out with it, so + that "the children of this event" is an index lookup. +2. **The listings.** `/relations` lists an event's children and `/threads` + lists a room's threads. +3. **The summaries.** Wherever an event is served, its children are + summarised into `unsigned.m.relations`, so a client drawing a timeline + knows a message has a thread under it, or has been edited, without + asking about every event. + +The code is in `internal/roomserver/relations.go` (the reads and the +summaries) and `internal/syncapi/relations.go` (the tokens), over the +relation reads of the room store in `internal/storage/relations.go`. + +## The index + +`roomserver_events` has two more columns, `relates_to_id` and `rel_type`, +taken from `content.m.relates_to.event_id` and `.rel_type` when both are +strings. Migration 0012 fills them in for the events already stored. An +`m.in_reply_to` on its own is not a relation in this sense: a reply points +at an event without being one of its children. + +A redaction clears both columns. The redaction algorithm strips the +content they came from, so a redacted thread reply leaves its thread, and +a redacted edit stops being one. + +## `GET /_matrix/client/v1/rooms/{roomID}/relations/{eventID}[/{relType}[/{eventType}]]` + +The children of one event, newest first by default. + +| Parameter | | +|---|---| +| `relType`, `eventType` | Narrow by relation type, then by event type. | +| `dir` | `b` (default) or `f`. | +| `from`, `to` | Pagination tokens. They are `/messages` tokens: a sync's `prev_batch` works as a `from`. | +| `limit` | 50 by default, 1000 at most. | +| `recurse` | Include the children's children, down to three levels. The response then carries `recursion_depth`: how deep it found anything. The filters apply at every level. | + +The parent has to be visible to the user, and the children are bounded to +what they may see, by the same visible range as a timeline read +([rooms.md](rooms.md)). A parent the user may not see is `404`, not `403`: +whether a particular event exists in a room is itself not theirs to know. + +## `GET /_matrix/client/v1/rooms/{roomID}/threads` + +The room's threads, the most recently active first. A thread is a root +that has at least one `m.thread` child; its activity is its newest child. +`include=participated` keeps the threads the user started or replied in. +Each root carries its `m.thread` summary. `from` takes the `next_batch` of +the previous page, a token of its own (`t`), because threads are +ordered by activity rather than by any one stream. + +A thread whose root this server does not hold — sent before this server +joined the room and never backfilled — is not listed: there is no event to +show it under. + +## Unread counts and receipts + +Each notification is filed under its thread, a threaded read receipt reads +only that thread (`thread_id: "main"` only the main timeline), and a sync +whose filter sets `unread_thread_notifications` gets the counts per thread. +How, and what stays a room total, is in [push.md](push.md). + +## Bundled aggregations + +Every event a timeline read returns — `/sync`, sliding sync, `/messages`, +`/context`, `GET /event`, `/relations` and `/threads` — gets +`unsigned.m.relations` when it has children: + +| Key | What | +|---|---| +| `m.thread` | `latest_event` (the newest reply, in full), `count`, and `current_user_participated` — true for a user who sent the root or any reply. | +| `m.replace` | The newest edit, in full. Only an edit from the original's sender counts, and only one of the same event type; a state event cannot be edited, and a redacted event's edits are not shown. | +| `m.reference` | `chunk`: the IDs of the events referencing it, oldest first, at most 50. | + +Annotations (reactions) are not summarised. The spec dropped their +aggregation, and every client counts reactions from the events themselves. + +The summaries are computed when the event is read, by one grouped query +over the batch of events being returned, bounded to what the user may +see: a user who has left the room sees a thread's count and latest reply +as of when they left. + +## What is deliberately missing + +| Thing | Why | +|---|---| +| Rejecting a thread whose root is itself a relation | The spec says a server *should*; Warp accepts it, and clients render it as an ordinary reply. | +| The relation filters of a room event filter (`related_by_rel_types`, `related_by_senders`) | Ignored, like every filter field Warp does not act on; the client gets a superset. | diff --git a/docs/search.md b/docs/search.md new file mode 100644 index 0000000..e9050f5 --- /dev/null +++ b/docs/search.md @@ -0,0 +1,79 @@ +# Search + +`POST /_matrix/client/v3/search` searches the rooms a user is in, or was +in, for words in messages, room names and topics. + +The roomserver decides where a user may look (`internal/roomserver/search.go`), +the store runs the query against a full-text index +(`internal/storage/sqlstore/relations.go`), and the sync API shapes the +response (`internal/syncapi/search.go`). + +## The index + +Each storage engine uses its own full-text search, and the drivers hand the +shared store the statements for it: + +| Driver | Index | Ranking | +|---|---|---| +| sqlite, pocketbase | an FTS5 table, tokenised with `unicode61 remove_diacritics 2`, so `cafe` finds `café` | `bm25` | +| postgres | a `tsvector` column with the `simple` configuration and a GIN index | `ts_rank` | + +`simple` lower-cases and splits words without stemming for any one +language, which suits a server whose rooms are in every language at once. +It keeps diacritics, so on PostgreSQL `cafe` does not find `café`. + +An event joins the index when it joins a timeline: `content.body` of an +`m.room.message`, `content.name` of `m.room.name`, `content.topic` of +`m.room.topic`. Outliers are not indexed; an outlier promoted into the +timeline is, unless it had already been redacted. A redaction removes the +event from the index, and purging a room removes the room's rows. +Migration 0012 indexes the events already stored. + +An encrypted message has no body the server can read and is never found. +That is what end-to-end encryption means; clients search encrypted rooms +themselves. + +## What a user may search + +Every room the user is joined to, or has left or been banned from, each +bounded by the same visible range as a timeline read +([rooms.md](rooms.md)). A user who left a room finds what was said while +they were in it and nothing since; a user who joined a room with `joined` +history visibility finds nothing from before they came. The bounds go into +the query itself, one clause per room, so a page is never short because +results it could not show were filtered out afterwards. + +## The query + +What the user typed is reduced to words — runs of letters and digits — +and each is matched as a prefix, all of them required. Both engines' +query languages have operators (FTS5's quotes, `NEAR` and column filters; +tsquery's `&`, `|`, `!`), and a search box is not the place to expose +them: a user searching for `C++ & Go` means the words. A term with no words +in it finds nothing. + +## Request and response + +| Request field | | +|---|---| +| `search_term` | Required. | +| `keys` | Any of `content.body`, `content.name`, `content.topic`; all three by default. | +| `filter` | A room event filter: `rooms`, `not_rooms`, `types`, `not_types`, `senders`, `not_senders`, and `limit` (10 by default, 100 at most). | +| `order_by` | `rank` (default) or `recent`. | +| `event_context` | `before_limit` and `after_limit` (5 each by default, 20 at most), and `include_profile` for the senders' names and avatars. | +| `include_state` | The current state of every room with a result. | +| `groupings.group_by` | `room_id` and/or `sender`. | + +The response carries `count` (every match, not just this page), +`highlights` (the words searched for), the `results` with their `rank`, and +`next_batch` when there is more. A ranked search pages by offset; a search +by recency pages by stream position, which keeps a page from repeating a +result when new messages arrive between two requests. + +## What is deliberately missing + +| Thing | Why | +|---|---| +| Ranking across engines | `rank` is only comparable within one search on one server. | +| Highlighting stemmed forms | Highlights are the words as typed. | +| Searching rooms by their contents over federation | Search covers what this server holds. | diff --git a/docs/spaces.md b/docs/spaces.md new file mode 100644 index 0000000..b4b2195 --- /dev/null +++ b/docs/spaces.md @@ -0,0 +1,85 @@ +# Spaces + +A space is a room whose create event says `"type": "m.space"`, and whose +`m.space.child` state events name the rooms in it. As with relations, the +authorisation rules know nothing of it: a child event is a pointer, and +the room it points at need not know. What the server provides is the walk. + +The code is in `internal/roomserver/hierarchy.go`. + +## `GET /_matrix/client/v1/rooms/{roomID}/hierarchy` + +The space's tree, breadth first, a page at a time. Each room is described +the way the public room directory describes it — name, topic, avatar, +alias, member count, join rule, room type, room version and encryption — +plus its `children_state`: the `m.space.child` events, each with its +`origin_server_ts`. + +| Parameter | | +|---|---| +| `max_depth` | How far below the root to go; `0` is the root alone. Capped at 50. | +| `suggested_only` | Follow only children marked `"suggested": true`. | +| `limit` | Rooms per page: 50 by default, 100 at most. | +| `from` | The previous page's `next_batch`. | + +Children are visited in the spec's order: those with a valid `order` (printable ASCII, at most 50 characters) by that string, then by when the child event was sent, then by room ID. +A child event whose `via` is missing or empty is how a space removes a +child — a state event cannot be deleted, only emptied — and is skipped. +Only a space's children are walked into: an ordinary room that happens to +hold an `m.space.child` event is not a space. + +Paging walks from the root again and skips what earlier pages returned. +That keeps the server free of per-walk state, and a walk is capped at 2000 +rooms whatever the page size, which bounds the cost of the last page of a +very large space. + +### Who sees what + +A room is shown to a user who: + +- is joined to it or invited to it; or +- may know about it anyway: its join rule is `public`, `knock` or + `knock_restricted`, or its history is world-readable; or +- is joined to one of the rooms a `restricted` room lets in. + +Everything else is left out, and so are its children. The root itself is +the exception: a space the user may not see is `403`. + +### Rooms this server is not in + +A child this server is not in is asked about over federation, of the +servers its `via` names, with `GET /_matrix/federation/v1/hierarchy`. The +answer is remembered for the rest of the walk. The other server has +already judged the room for this server as a whole; Warp then checks a +restricted room's allow list against its own user. A child that no server +will describe is left out. With federation off, rooms this server is not +in are simply not shown. + +## `GET /_matrix/federation/v1/hierarchy/{roomID}` + +Another server walking one of Warp's spaces: the room and the children +Warp is in, each judged for the asking server. A room is shown to a server +when anybody could be shown it (see above), when it is restricted — with +`allowed_room_ids`, so the asking server can check its own users — or +when a user of that server is joined or invited. Children it may not see +are listed in `inaccessible_children`. A root it may not see is `404`, +which is also the answer for a room Warp is not in. + +## `GET /_matrix/client/v1/room_summary/{roomIDOrAlias}` + +One room's face, for a link preview or an invite: the same fields as a +hierarchy room without its children, plus the user's `membership` +(`leave` for a user who was never in it). An alias is resolved first, and +the servers it names are added to `?via=`. The same visibility rules apply; +a room the user may not see is `403`. + +The endpoint is in the spec from v1.15. Warp still advertises v1.14, +because v1.15 also brought requirements Warp does not meet yet. + +## What is deliberately missing + +| Thing | Why | +|---|---| +| Unauthenticated `room_summary` | The spec allows it for guests; Warp asks for a token. | +| Caching other servers' hierarchy answers between walks | Each walk asks again; a space with many remote children costs a federation request per child per page. | +| `m.space.parent` checks | Parent events are state like any other, and are not used to validate children. | diff --git a/docs/sync.md b/docs/sync.md index 26bfde5..2e56c1b 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -256,6 +256,14 @@ So the test is whether the response has anything in it, with an initial sync as the exception: a client that has just logged in needs an answer whatever is in it. +"Initial" means the client sent no `?since=`, not that its token is all +zeroes. On a server whose streams have not moved yet, `s0` is every +client's token, and taking it for a first sync made every long poll +return at once. The push rules a first sync carries are sent on the +client's first sync for the same reason, not on any sync whose account +data position is zero: before anybody had written account data, that was +every sync, and each one counted as news. + The same trap caught presence. A client says it is still online by syncing, so `?set_presence=` makes every sync a presence write; a stream that moved each time would wake every peer's long poll, which would sync, which would @@ -280,11 +288,12 @@ What Warp acts on: tuples, not a window of a stream, so there is no index to push it to. - `room.rooms` / `not_rooms`, at both levels. - `room.timeline.lazy_load_members`. +- `room.timeline.unread_thread_notifications`: the unread counts split by + thread; see [push.md](push.md). - `room.include_leave`. - `presence` and `account_data` types and senders. -What it parses and does not act on: `event_fields`, `contains_url`, -`unread_thread_notifications`. A field Warp ignores is **not** an error — a +What it parses and does not act on: `event_fields`, `contains_url`. A field Warp ignores is **not** an error — a server is allowed to return a superset of what a filter asked for, and refusing the filter is the one outcome that would stop a client syncing at all. `event_format: federation` *is* refused: that is a server's view of an @@ -400,16 +409,120 @@ its own response type for that reason. Reporting it as a room event with empty fields would be worse: a client would see an event with no sender and no timestamp and have to decide what that meant. +## Simplified Sliding Sync + +`POST /_matrix/client/unstable/org.matrix.simplified_msc3575/sync` is +MSC4186, which Element X and every client on the matrix-rust-sdk use when +`/versions` advertises `org.matrix.simplified_msc3575` — as Warp's does. + +A classic initial sync sends every room before a client can draw +anything, which for a large account takes minutes. Sliding sync turns that +round: the client asks for *lists* — "my rooms, most recently active +first, positions 0 to 19" — and for the few pieces of state it needs to +draw each one, and is sent those rooms and a count of the rest. As the +user scrolls the client widens the window; as rooms become active they +move into it and are sent. + +The code is `internal/syncapi/sliding.go` and `slidingconn.go`, over a few +roomserver reads in `internal/roomserver/sliding.go`. + +### Connections + +What a client has been sent depends on what it asked for, so a token +cannot just be a position in each stream. The server remembers, per +connection — one user, one device, one `conn_id` — which rooms it has sent +and up to which position, and `pos` names one of those memories. A room is +sent once in full (`"initial": true`) and then only when it changes; a room +that leaves the window and comes back is not sent again unless something +happened to it. + +The memory is in the process and is lost on restart. An unknown `pos` — +expired after 30 minutes idle, from before a restart, or one of the +connection's older positions beyond the last three — is answered with +`400 M_UNKNOWN_POS`, and the client starts a new connection. That is how +the protocol is meant to recover. A request without `pos` starts its +connection over. A device holds at most ten connections; the oldest goes +first. + +Request parameters are not sticky: every request says what it wants. + +### Lists + +A list holds the rooms the user is joined to, invited to or knocking on, +plus rooms they left or were banned from while the connection was open, +which is how a client hears that a room it is showing is gone. Rooms are +sorted by `bump_stamp`, the position of the last message, encrypted +message, sticker, call invite, poll or live location in the room (not +membership changes, reactions or receipts, which would keep a busy room on +top for reasons nobody could see). An invite, a knock or a room the user +has left is bumped by their own membership event, because they cannot see +what happens in it. + +| Filter | | +|---|---| +| `is_dm` | rooms in the user's `m.direct` account data | +| `is_invite` | | +| `is_encrypted` | an `m.room.encryption` in the room's state | +| `room_types`, `not_room_types` | the create event's `type`; `null` is an ordinary room | +| `spaces` | children of the named spaces this server is in | +| `tags`, `not_tags` | the room's `m.tag` account data | + +`ranges` are inclusive windows into the sorted list. The response gives +each list's `count`, and the rooms in any window, merged with the room +subscriptions. + +### Rooms + +`room_subscriptions` names rooms directly, whatever list they are or are +not in: the room a client has open. A room in several lists and +subscriptions is sent with the largest `timeline_limit` and the union of +the `required_state`. + +`required_state` is a list of `[type, state_key]`. `*` is a wildcard in +either place, `$ME` is the user, and `["m.room.member", "$LAZY"]` is the +members who sent something in the timeline, plus the user. Only those +tuples are read: the room's state is read as a set of keys first and the +event bodies after, so a room with ten thousand members costs one +member event, not ten thousand. On an incremental response it is the +tuples that changed since the room was last sent. + +Each room carries `name` and `avatar`, `heroes` (with names and avatars) +when it has no name, `is_dm`, `joined_count`, `invited_count`, the unread +counts, `bump_stamp`, and the `timeline` with `prev_batch` (a `/messages` +token), `limited` and `num_live` (how many of the timeline's events are +new since the previous response). An invite is its `invite_state`, a knock +its `knock_state`, and neither has a timeline. + +### Extensions + +| Extension | What | +|---|---| +| `to_device` | The device's queued messages, 100 at a time by default. It has a `since` token of its own; coming back with it acknowledges the batch. | +| `e2ee` | `device_lists`, `device_one_time_keys_count`, `device_unused_fallback_key_types`, worked out as for `/sync`. | +| `account_data` | Global, and per room for the rooms in the response's lists and subscriptions — all of it for a room sent in full. | +| `receipts` | Per room, the receipts since the last response; all of them for a room sent in full. | +| `typing` | Per room, the typing list when it changed — including to empty, so a client stops showing somebody typing. | + +`lists` and `rooms` narrow the rooms a per-room extension covers. Naming +any list the request has is every room in the windows: the windows are +merged before the extensions run. + +### Long polling + +A response is returned at once on a new connection; otherwise only when a +room is sent, a list's count moved, or an extension has something to say. +Until then it waits on the same notifier as `/sync`. + ## What is deliberately missing -| Thing | Needs | Phase | -|---|---|---| -| `unread_thread_notifications` | threads | 10 | -| Remote presence | presence over federation | — | -| Threads, and threaded receipts beyond storing `thread_id` | threads | 10 | -| `contains_url`, `event_fields` | nothing; they are parsed and ignored | — | -| Per-event history visibility | state storage (open question 1) | — | -| Simplified Sliding Sync (MSC4186) | — | 10 | +| Thing | Needs | +|---|---| +| Remote presence | presence over federation | +| `contains_url`, `event_fields` | nothing; they are parsed and ignored | +| Per-event history visibility | state storage (open question 1) | + +Threads, relations and the summaries every timeline event carries in +`unsigned.m.relations` are in [relations.md](relations.md). `device_lists` is sent as two empty lists rather than left out: a client reads a missing one as a malformed response, where two empty lists say @@ -417,7 +530,9 @@ reads a missing one as a malformed response, where two empty lists say that ride along with them, are in [e2ee.md](e2ee.md). Each joined room carries `unread_notifications` from the push server, -whose counts are the seventh stream, and the global account data carries +whose counts are the seventh stream — the main timeline's alone, beside an +`unread_thread_notifications` map of each thread's, when the filter sets +`unread_thread_notifications` — and the global account data carries `m.push_rules`, rendered over the server defaults for a user who never changed a rule. Both are described in [push.md](push.md). A build with no push server leaves the counts out entirely rather than sending zeroes, @@ -429,6 +544,9 @@ is not. | Endpoint | Notes | |---|---| | `GET /sync` | Long-polls. `?since=`, `?filter=`, `?timeout=`, `?full_state=`, `?set_presence=`. | +| `POST /_matrix/client/unstable/org.matrix.simplified_msc3575/sync` | Simplified Sliding Sync; see below. `?pos=`, `?timeout=`, `?set_presence=`. | +| `GET /_matrix/client/v1/rooms/{id}/relations/{eventId}` · `GET /_matrix/client/v1/rooms/{id}/threads` | See [relations.md](relations.md). | +| `POST /search` | See [search.md](search.md). | | `GET /rooms/{id}/messages` | `?dir=` (`b` default), `?from=`, `?to=`, `?limit=`, `?filter=`. No `end` at the end of the room. | | `GET /rooms/{id}/context/{eventId}` | `?limit=`, `?filter=`. Half the budget each side, the odd one backwards. | | `POST /user/{userId}/filter` | Returns an opaque ID. | diff --git a/server/cmd/warp-loadtest/main.go b/server/cmd/warp-loadtest/main.go new file mode 100644 index 0000000..ccc6037 --- /dev/null +++ b/server/cmd/warp-loadtest/main.go @@ -0,0 +1,514 @@ +// Command warp-loadtest drives a homeserver the way a crowd of clients +// would, and reports how it held up. +// +// It registers users, creates rooms and has every user join them, then for +// the length of the run each user sends messages at a steady rate while +// syncing in a long-poll loop, the way a client sits in /sync. Every message +// carries the time it was sent, so a user's sync can measure how long the +// server took to deliver what somebody else said — the number a user of a +// chat server actually feels — alongside the latency of each request. +// +// It speaks the plain Client-Server API and works against any homeserver, +// not only Warp. Point it at a test server: it creates accounts and rooms +// it never cleans up. Rate limiting on the server has to be off, or the +// run measures the rate limiter. +// +// warp-loadtest -url http://localhost:8008 -users 50 -rooms 10 -rate 0.5 -duration 2m +package main + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/signal" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "text/tabwriter" + "time" +) + +type options struct { + baseURL string + users int + rooms int + rate float64 + duration time.Duration + prefix string + password string + regToken string + sliding bool + setupParallel int +} + +func main() { + var o options + flag.StringVar(&o.baseURL, "url", "http://localhost:8008", "the homeserver's client API base URL") + flag.IntVar(&o.users, "users", 20, "how many users to register") + flag.IntVar(&o.rooms, "rooms", 5, "how many rooms to create; every user joins every room") + flag.Float64Var(&o.rate, "rate", 0.2, "messages per second per user") + flag.DurationVar(&o.duration, "duration", time.Minute, "how long to send for") + flag.StringVar(&o.prefix, "prefix", "", "localpart prefix for the users (default: random)") + flag.StringVar(&o.password, "password", "loadtest-password-1234", "the users' password") + flag.StringVar(&o.regToken, "registration-token", "", "a registration token, for a server that requires one") + flag.BoolVar(&o.sliding, "sliding", false, "sync with Simplified Sliding Sync instead of /sync") + flag.IntVar(&o.setupParallel, "setup-parallelism", 8, "concurrent requests while registering and joining") + flag.Parse() + + if o.prefix == "" { + var b [3]byte + _, _ = rand.Read(b[:]) + o.prefix = "lt" + hex.EncodeToString(b[:]) + } + o.baseURL = strings.TrimRight(o.baseURL, "/") + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + if err := run(ctx, o); err != nil { + fmt.Fprintln(os.Stderr, "warp-loadtest:", err) + os.Exit(1) + } +} + +func run(ctx context.Context, o options) error { + stats := newStats() + c := &client{base: o.baseURL, http: &http.Client{Timeout: 90 * time.Second}, stats: stats} + + fmt.Printf("registering %d users as %s-N on %s\n", o.users, o.prefix, o.baseURL) + users := make([]*user, o.users) + if err := parallel(ctx, o.users, o.setupParallel, func(i int) error { + u, err := c.register(ctx, fmt.Sprintf("%s-%d", o.prefix, i), o.password, o.regToken) + users[i] = u + return err + }); err != nil { + return err + } + + fmt.Printf("creating %d rooms and joining every user to each\n", o.rooms) + rooms := make([]string, o.rooms) + for i := range rooms { + roomID, err := c.createRoom(ctx, users[0], fmt.Sprintf("load test %s #%d", o.prefix, i)) + if err != nil { + return err + } + rooms[i] = roomID + } + if err := parallel(ctx, len(users)-1, o.setupParallel, func(i int) error { + for _, roomID := range rooms { + if err := c.join(ctx, users[i+1], roomID); err != nil { + return err + } + } + return nil + }); err != nil { + return err + } + + fmt.Printf("sending %.2f msg/s per user for %s (%s)\n", o.rate, o.duration, map[bool]string{true: "sliding sync", false: "/sync"}[o.sliding]) + + var wg sync.WaitGroup + syncCtx, stopSync := context.WithCancel(ctx) + for _, u := range users { + wg.Add(1) + go func() { + defer wg.Done() + if o.sliding { + c.slidingLoop(syncCtx, u, len(rooms)) + } else { + c.syncLoop(syncCtx, u) + } + }() + } + // The syncs need a moment to reach the top of the stream before the + // first message, or the first delivery latencies would include the + // initial sync. + time.Sleep(2 * time.Second) + + runCtx, cancel := context.WithTimeout(ctx, o.duration) + defer cancel() + var senders sync.WaitGroup + start := time.Now() + for i, u := range users { + senders.Add(1) + go func() { + defer senders.Done() + c.sendLoop(runCtx, u, rooms, o.rate, float64(i)/float64(len(users))) + }() + } + senders.Wait() + elapsed := time.Since(start) + + // Give the last messages time to be delivered before the syncs stop. + time.Sleep(3 * time.Second) + stopSync() + wg.Wait() + + stats.report(os.Stdout, elapsed) + return nil +} + +// parallel runs fn for 0..n-1, at most width at once, stopping at the +// first error. +func parallel(ctx context.Context, n, width int, fn func(int) error) error { + sem := make(chan struct{}, max(width, 1)) + var ( + wg sync.WaitGroup + mu sync.Mutex + firstErr error + ) + for i := range n { + if ctx.Err() != nil { + break + } + sem <- struct{}{} + wg.Add(1) + go func() { + defer func() { <-sem; wg.Done() }() + if err := fn(i); err != nil { + mu.Lock() + if firstErr == nil { + firstErr = err + } + mu.Unlock() + } + }() + } + wg.Wait() + if firstErr != nil { + return firstErr + } + return ctx.Err() +} + +type user struct { + id string + token string +} + +type client struct { + base string + http *http.Client + stats *stats + txn atomic.Int64 +} + +// do sends one request, timing it under op, and decodes the response into +// out when there is one. +func (c *client) do(ctx context.Context, op, method, path, token string, body, out any) (int, error) { + var reader io.Reader + if body != nil { + raw, err := json.Marshal(body) + if err != nil { + return 0, err + } + reader = bytes.NewReader(raw) + } + req, err := http.NewRequestWithContext(ctx, method, c.base+path, reader) + if err != nil { + return 0, err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + start := time.Now() + resp, err := c.http.Do(req) + if err != nil { + if ctx.Err() == nil { + c.stats.fail(op) + } + return 0, err + } + defer func() { _ = resp.Body.Close() }() + raw, err := io.ReadAll(resp.Body) + c.stats.observe(op, time.Since(start), resp.StatusCode < 400 || resp.StatusCode == http.StatusUnauthorized && op == "register") + if err != nil { + return resp.StatusCode, err + } + if out != nil && len(raw) > 0 { + _ = json.Unmarshal(raw, out) + } + if resp.StatusCode >= 400 && (resp.StatusCode != http.StatusUnauthorized || op != "register") { + return resp.StatusCode, fmt.Errorf("%s %s: %d %s", method, path, resp.StatusCode, strings.TrimSpace(string(raw))) + } + return resp.StatusCode, nil +} + +// register walks the user-interactive registration flow, answering the +// dummy and registration-token stages. +func (c *client) register(ctx context.Context, localpart, password, token string) (*user, error) { + body := map[string]any{"username": localpart, "password": password, "inhibit_login": false} + var challenge struct { + Session string `json:"session"` + Flows []struct { + Stages []string `json:"stages"` + } `json:"flows"` + UserID string `json:"user_id"` + AccessToken string `json:"access_token"` + } + if _, err := c.do(ctx, "register", http.MethodPost, "/_matrix/client/v3/register", "", body, &challenge); err != nil { + return nil, err + } + if challenge.AccessToken != "" { + return &user{id: challenge.UserID, token: challenge.AccessToken}, nil + } + if len(challenge.Flows) == 0 { + return nil, errors.New("register: the server offered no flow") + } + for _, stage := range challenge.Flows[0].Stages { + auth := map[string]any{"type": stage, "session": challenge.Session} + switch stage { + case "m.login.dummy": + case "m.login.registration_token": + if token == "" { + return nil, errors.New("register: the server wants a registration token; pass -registration-token") + } + auth["token"] = token + default: + return nil, fmt.Errorf("register: cannot complete the %s stage", stage) + } + body["auth"] = auth + var next struct { + UserID string `json:"user_id"` + AccessToken string `json:"access_token"` + } + if _, err := c.do(ctx, "register", http.MethodPost, "/_matrix/client/v3/register", "", body, &next); err != nil { + return nil, err + } + if next.AccessToken != "" { + return &user{id: next.UserID, token: next.AccessToken}, nil + } + } + return nil, errors.New("register: the flow ended without an access token") +} + +func (c *client) createRoom(ctx context.Context, u *user, name string) (string, error) { + var out struct { + RoomID string `json:"room_id"` + } + _, err := c.do(ctx, "createRoom", http.MethodPost, "/_matrix/client/v3/createRoom", u.token, + map[string]any{"name": name, "preset": "public_chat"}, &out) + return out.RoomID, err +} + +func (c *client) join(ctx context.Context, u *user, roomID string) error { + _, err := c.do(ctx, "join", http.MethodPost, "/_matrix/client/v3/join/"+url.PathEscape(roomID), u.token, map[string]any{}, nil) + return err +} + +// sentMarker prefixes a message body with the time it was sent. +const sentMarker = "warp-loadtest sent=" + +// sendLoop sends at a steady rate until the context ends. Users start +// staggered across the first interval — phase is this user's place in it, +// from 0 to 1 — so that the load is a flow rather than a pulse. +func (c *client) sendLoop(ctx context.Context, u *user, rooms []string, rate, phase float64) { + if rate <= 0 { + return + } + interval := time.Duration(float64(time.Second) / rate) + select { + case <-ctx.Done(): + return + case <-time.After(time.Duration(phase * float64(interval))): + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + n := 0 + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + roomID := rooms[n%len(rooms)] + n++ + body := sentMarker + strconv.FormatInt(time.Now().UnixNano(), 10) + path := fmt.Sprintf("/_matrix/client/v3/rooms/%s/send/m.room.message/lt%d", + url.PathEscape(roomID), c.txn.Add(1)) + _, _ = c.do(ctx, "send", http.MethodPut, path, u.token, map[string]any{"msgtype": "m.text", "body": body}, nil) + } +} + +// syncLoop sits in /sync, measuring how long each message took to arrive. +func (c *client) syncLoop(ctx context.Context, u *user) { + since := "" + for ctx.Err() == nil { + path := "/_matrix/client/v3/sync?timeout=30000" + if since != "" { + path += "&since=" + url.QueryEscape(since) + } + var resp struct { + NextBatch string `json:"next_batch"` + Rooms struct { + Join map[string]struct { + Timeline struct { + Events []timelineEvent `json:"events"` + } `json:"timeline"` + } `json:"join"` + } `json:"rooms"` + } + op := "sync" + if since == "" { + op = "initial sync" + } + if _, err := c.do(ctx, op, http.MethodGet, path, u.token, nil, &resp); err != nil { + if ctx.Err() == nil { + time.Sleep(time.Second) + } + continue + } + received := time.Now() + if since != "" { + for _, room := range resp.Rooms.Join { + c.deliveries(u, room.Timeline.Events, received) + } + } + since = resp.NextBatch + } +} + +// slidingLoop is syncLoop on Simplified Sliding Sync. +func (c *client) slidingLoop(ctx context.Context, u *user, rooms int) { + const path = "/_matrix/client/unstable/org.matrix.simplified_msc3575/sync" + body := map[string]any{ + "lists": map[string]any{"all": map[string]any{ + "ranges": [][2]int{{0, max(rooms-1, 0)}}, "timeline_limit": 20, + "required_state": [][]string{{"m.room.name", ""}}, + }}, + } + pos := "" + for ctx.Err() == nil { + q := "?timeout=30000" + if pos != "" { + q += "&pos=" + url.QueryEscape(pos) + } + var resp struct { + Pos string `json:"pos"` + Rooms map[string]struct { + Timeline []timelineEvent `json:"timeline"` + } `json:"rooms"` + } + op := "sliding sync" + if pos == "" { + op = "initial sliding sync" + } + status, err := c.do(ctx, op, http.MethodPost, path+q, u.token, body, &resp) + if err != nil { + if status == http.StatusBadRequest { + pos = "" // M_UNKNOWN_POS: start again + } + if ctx.Err() == nil { + time.Sleep(time.Second) + } + continue + } + received := time.Now() + if pos != "" { + for _, room := range resp.Rooms { + c.deliveries(u, room.Timeline, received) + } + } + pos = resp.Pos + } +} + +type timelineEvent struct { + Type string `json:"type"` + Sender string `json:"sender"` + Content struct { + Body string `json:"body"` + } `json:"content"` +} + +// deliveries records how long each message from somebody else took to +// reach this user. +func (c *client) deliveries(u *user, events []timelineEvent, received time.Time) { + for _, e := range events { + if e.Type != "m.room.message" || e.Sender == u.id { + continue + } + raw, ok := strings.CutPrefix(e.Content.Body, sentMarker) + if !ok { + continue + } + sent, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + continue + } + c.stats.observe("delivery", received.Sub(time.Unix(0, sent)), true) + } +} + +// stats collects latencies per operation. +type stats struct { + mu sync.Mutex + ops map[string][]time.Duration + errors map[string]int +} + +func newStats() *stats { + return &stats{ops: map[string][]time.Duration{}, errors: map[string]int{}} +} + +func (s *stats) observe(op string, d time.Duration, ok bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.ops[op] = append(s.ops[op], d) + if !ok { + s.errors[op]++ + } +} + +func (s *stats) fail(op string) { + s.mu.Lock() + defer s.mu.Unlock() + s.errors[op]++ +} + +func (s *stats) report(w io.Writer, elapsed time.Duration) { + s.mu.Lock() + defer s.mu.Unlock() + + names := make([]string, 0, len(s.ops)) + for name := range s.ops { + names = append(names, name) + } + slices.Sort(names) + + fmt.Fprintf(w, "\nran for %s\n\n", elapsed.Round(time.Millisecond)) + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', tabwriter.AlignRight) + fmt.Fprintln(tw, "operation\tcount\terrors\trate/s\tp50\tp90\tp99\tmax\t") + for _, name := range names { + d := s.ops[name] + slices.Sort(d) + rate := float64(len(d)) / elapsed.Seconds() + fmt.Fprintf(tw, "%s\t%d\t%d\t%.1f\t%s\t%s\t%s\t%s\t\n", name, len(d), s.errors[name], rate, + pct(d, 0.5), pct(d, 0.9), pct(d, 0.99), pct(d, 1)) + } + _ = tw.Flush() + fmt.Fprintln(w, "\nsync latencies include the long-poll wait; delivery is send-to-receipt across users.") +} + +func pct(sorted []time.Duration, q float64) string { + if len(sorted) == 0 { + return "-" + } + i := int(q*float64(len(sorted))+0.5) - 1 + i = max(0, min(i, len(sorted)-1)) + return sorted[i].Round(100 * time.Microsecond).String() +} diff --git a/server/cmd/warp/createaccount.go b/server/cmd/warp/createaccount.go index d588097..0120e4f 100644 --- a/server/cmd/warp/createaccount.go +++ b/server/cmd/warp/createaccount.go @@ -102,13 +102,7 @@ func runCreateAccount(ctx context.Context, args []string) error { // openDatabase opens the storage driver named in the config. func openDatabase(ctx context.Context, cfg *config.Config, log *slog.Logger) (storage.Database, error) { - return storage.Open(ctx, storage.Config{ - Driver: cfg.Database.Driver, - ConnectionString: cfg.Database.ConnectionString, - MaxOpenConns: cfg.Database.MaxOpenConns, - MaxIdleConns: cfg.Database.MaxIdleConns, - ConnMaxLifetime: cfg.Database.ConnMaxLifetime, - }, log) + return storage.Open(ctx, cfg.Database.Storage(), log) } // resolvePassword settles where the password comes from. diff --git a/server/go.mod b/server/go.mod index d525794..91fa60c 100644 --- a/server/go.mod +++ b/server/go.mod @@ -4,7 +4,13 @@ go 1.26.4 require ( github.com/jackc/pgx/v5 v5.11.0 + github.com/pocketbase/dbx v1.12.0 + github.com/pocketbase/pocketbase v0.39.11 github.com/prometheus/client_golang v1.25.0 + go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 + go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/trace v1.47.0 golang.org/x/crypto v0.58.0 golang.org/x/image v0.46.0 golang.org/x/sync v0.24.0 @@ -14,24 +20,48 @@ require ( ) require ( + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/disintegration/imaging v1.6.2 // indirect + github.com/domodwyer/mailyak/v3 v3.6.2 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/fatih/color v1.19.0 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect + github.com/gabriel-vasile/mimetype v1.4.13 // indirect + github.com/ganigeorgiev/fexpr v0.6.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/kr/text v0.2.0 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/pocketbase/ozzo-validation/v4 v4.3.0 // indirect github.com/prometheus/client_model v0.6.3 // indirect github.com/prometheus/common v0.72.0 // indirect github.com/prometheus/procfs v0.22.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rogpeppe/go-internal v1.16.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/otel/metric v1.47.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect + golang.org/x/net v0.60.0 // indirect + golang.org/x/oauth2 v0.37.0 // indirect golang.org/x/sys v0.49.0 // indirect golang.org/x/text v0.43.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.12 // indirect modernc.org/libc v1.77.1 // indirect modernc.org/mathutil v1.7.1 // indirect diff --git a/server/go.sum b/server/go.sum index e13ad3c..b15bcff 100644 --- a/server/go.sum +++ b/server/go.sum @@ -1,17 +1,49 @@ +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= +github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8= +github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= +github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/ganigeorgiev/fexpr v0.6.0 h1:Fza3O/QMBKEudUvxV862qe6GjxM60GJjjKytdp+VQus= +github.com/ganigeorgiev/fexpr v0.6.0/go.mod h1:RyGiGqmeXhEQ6+mlGdnUleLHgtzzu/VGO2WtJkF5drE= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -24,12 +56,14 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -37,6 +71,12 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pocketbase/dbx v1.12.0 h1:/oLErM+A0b4xI0PWTGPqSDVjzix48PqI/bng2l0PzoA= +github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs= +github.com/pocketbase/ozzo-validation/v4 v4.3.0 h1:uKBDVma7bZqgR2a6AwE+k9hkuDFfiZMpBHQdZ1z3iQs= +github.com/pocketbase/ozzo-validation/v4 v4.3.0/go.mod h1:6XNjSTw/Jb2F8LOkKO3oyzIWExbrGiYoS4uVxVwz90g= +github.com/pocketbase/pocketbase v0.39.11 h1:cl/Kh13ukof/4BAEku3OozYrLl85M5/bmH62Ny4szFc= +github.com/pocketbase/pocketbase v0.39.11/go.mod h1:5CaCvp/52fZJ5/qyYsqpCGyVue/kjez889cQAATD2cY= github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKqbYtNOXuiK0= github.com/prometheus/client_golang v1.25.0/go.mod h1:j2X/4/kaQHLPnmyEU8zjmkndo1T317VpCIQEml8LTKk= github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= @@ -49,38 +89,83 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ= golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= +golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/server/internal/appservice/client.go b/server/internal/appservice/client.go index c4a2bd6..e70a453 100644 --- a/server/internal/appservice/client.go +++ b/server/internal/appservice/client.go @@ -13,6 +13,8 @@ import ( "strconv" "strings" "time" + + "github.com/dariuszjaniak/warp/server/internal/tracing" ) // Errors of the Application Service API client. The ping endpoint reports @@ -57,6 +59,7 @@ type Client struct { func NewClient(timeout time.Duration) *Client { return &Client{ http: &http.Client{ + Transport: tracing.Transport(nil), // A service is reached at the URL its registration names and // nowhere else: a redirect would take the hs_token with it. CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, diff --git a/server/internal/clientapi/clientapi.go b/server/internal/clientapi/clientapi.go index 3c74099..3ccc9f2 100644 --- a/server/internal/clientapi/clientapi.go +++ b/server/internal/clientapi/clientapi.go @@ -313,9 +313,19 @@ func (c *ClientAPI) registerSyncRoutes(r *httputil.Router) { } c.authed(r, "GET "+v3+"/sync", c.limitSync, c.handleSync) + c.authed(r, "POST "+slidingSyncPath, c.limitSync, c.handleSlidingSync) c.authed(r, "GET "+v3+"/rooms/{roomID}/messages", c.limitDefault, c.handleMessages) c.authed(r, "GET "+v3+"/rooms/{roomID}/context/{eventID}", c.limitDefault, c.handleContext) + // Relations and threads, which the spec introduced at v1. + c.authed(r, "GET "+v1+"/rooms/{roomID}/relations/{eventID}", c.limitDefault, c.handleRelations) + c.authed(r, "GET "+v1+"/rooms/{roomID}/relations/{eventID}/{relType}", c.limitDefault, c.handleRelations) + c.authed(r, "GET "+v1+"/rooms/{roomID}/relations/{eventID}/{relType}/{eventType}", + c.limitDefault, c.handleRelations) + c.authed(r, "GET "+v1+"/rooms/{roomID}/threads", c.limitDefault, c.handleThreads) + + c.authed(r, "POST "+v3+"/search", c.limitDefault, c.handleSearch) + // Filters. c.authed(r, "POST "+v3+"/user/{userID}/filter", c.limitDefault, c.handlePostFilter) c.authed(r, "GET "+v3+"/user/{userID}/filter/{filterID}", c.limitDefault, c.handleGetFilter) @@ -387,6 +397,10 @@ func (c *ClientAPI) registerRoomRoutes(r *httputil.Router) { c.limitDefault, c.handleGetJoinedMembers) c.authed(r, "GET "+v3+"/joined_rooms", c.limitDefault, c.handleGetJoinedRooms) + // Spaces, and the summary of a room the user may not be in. + c.authed(r, "GET "+v1+"/rooms/{roomID}/hierarchy", c.limitDefault, c.handleHierarchy) + c.authed(r, "GET "+v1+"/room_summary/{roomIDOrAlias}", c.limitDefault, c.handleRoomSummary) + // The directory. Resolving an alias and reading the public room list // need no token: a client handed a link has to be able to follow it // before it decides whether to log in. diff --git a/server/internal/clientapi/relations.go b/server/internal/clientapi/relations.go new file mode 100644 index 0000000..1063016 --- /dev/null +++ b/server/internal/clientapi/relations.go @@ -0,0 +1,88 @@ +package clientapi + +import ( + "net/http" + + "github.com/dariuszjaniak/warp/server/internal/httputil" + "github.com/dariuszjaniak/warp/server/internal/storage" + "github.com/dariuszjaniak/warp/server/internal/syncapi" + "github.com/dariuszjaniak/warp/server/internal/userapi" +) + +// handleRelations implements +// GET /_matrix/client/v1/rooms/{roomID}/relations/{eventID}[/{relType}[/{eventType}]]. +// +// The three routes share the handler; the narrower ones only fill in more +// of the path. +func (c *ClientAPI) handleRelations(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + roomID, err := pathRoomID(r) + if err != nil { + return err + } + eventID := r.PathValue("eventID") + if eventID == "" { + return httputil.MissingParam("An event ID is required") + } + query := r.URL.Query() + + direction := storage.DirectionBackward + switch query.Get("dir") { + case "f": + direction = storage.DirectionForward + case "b", "": + default: + return httputil.InvalidParam("dir must be \"b\" or \"f\"") + } + limit, err := intParam(query.Get("limit")) + if err != nil { + return err + } + recurse, err := boolParam(query.Get("recurse")) + if err != nil { + return err + } + + resp, err := c.sync.Relations(r.Context(), syncapi.RelationsRequest{ + RoomID: roomID, + EventID: eventID, + User: auth.UserID, + RelType: r.PathValue("relType"), + EventType: r.PathValue("eventType"), + From: query.Get("from"), + To: query.Get("to"), + Direction: direction, + Limit: limit, + Recurse: recurse, + }) + if err != nil { + return syncError(err) + } + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} + +// handleThreads implements GET /_matrix/client/v1/rooms/{roomID}/threads. +func (c *ClientAPI) handleThreads(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + roomID, err := pathRoomID(r) + if err != nil { + return err + } + query := r.URL.Query() + limit, err := intParam(query.Get("limit")) + if err != nil { + return err + } + + resp, err := c.sync.Threads(r.Context(), syncapi.ThreadsRequest{ + RoomID: roomID, + User: auth.UserID, + Include: query.Get("include"), + From: query.Get("from"), + Limit: limit, + }) + if err != nil { + return syncError(err) + } + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} diff --git a/server/internal/clientapi/relations_test.go b/server/internal/clientapi/relations_test.go new file mode 100644 index 0000000..a3fd93b --- /dev/null +++ b/server/internal/clientapi/relations_test.go @@ -0,0 +1,306 @@ +package clientapi_test + +import ( + "fmt" + "net/http" + "net/url" + "testing" + "time" +) + +// Relations and threads: what /relations lists, what /threads lists, and +// the summaries a parent carries in unsigned.m.relations wherever it is +// served. + +// sendEvent sends an event of any type and returns its ID. +func (ts *testServer) sendEvent(acc account, roomID, eventType string, content map[string]any) string { + ts.t.Helper() + path := fmt.Sprintf("/_matrix/client/v3/rooms/%s/send/%s/%s", + url.PathEscape(roomID), url.PathEscape(eventType), fmt.Sprintf("txn-%d", time.Now().UnixNano())) + rec := ts.put(path, acc.AccessToken, content) + expectStatus(ts.t, rec, http.StatusOK) + eventID, _ := jsonBody(ts.t, rec)["event_id"].(string) + if eventID == "" { + ts.t.Fatalf("send returned no event ID: %s", rec.Body.String()) + } + return eventID +} + +// threadReply sends a message in a thread. +func (ts *testServer) threadReply(acc account, roomID, rootID, body string) string { + ts.t.Helper() + return ts.sendEvent(acc, roomID, "m.room.message", map[string]any{ + "msgtype": "m.text", "body": body, + "m.relates_to": map[string]any{"rel_type": "m.thread", "event_id": rootID}, + }) +} + +type relationsResult struct { + Chunk []syncEvent `json:"chunk"` + NextBatch string `json:"next_batch"` + RecursionDepth int `json:"recursion_depth"` +} + +func (ts *testServer) relations(acc account, roomID, path string, query url.Values) relationsResult { + ts.t.Helper() + full := "/_matrix/client/v1/rooms/" + url.PathEscape(roomID) + "/relations/" + path + if encoded := query.Encode(); encoded != "" { + full += "?" + encoded + } + rec := ts.get(full, acc.AccessToken) + expectStatus(ts.t, rec, http.StatusOK) + return decode[relationsResult](ts.t, rec) +} + +// threadSummaryOf digs unsigned.m.relations.m.thread out of an event. +func threadSummaryOf(t *testing.T, event map[string]any) map[string]any { + t.Helper() + unsigned, _ := event["unsigned"].(map[string]any) + relations, _ := unsigned["m.relations"].(map[string]any) + summary, _ := relations["m.thread"].(map[string]any) + return summary +} + +func TestRelationsListsChildrenNewestFirst(t *testing.T) { + ts := newTestServer(t, nil) + alice := ts.register("alice", "correct-horse-battery-staple") + roomID := ts.createRoom(alice, nil) + + root := ts.sendMessage(alice, roomID, "root") + r1 := ts.threadReply(alice, roomID, root, "one") + r2 := ts.threadReply(alice, roomID, root, "two") + reaction := ts.sendEvent(alice, roomID, "m.reaction", map[string]any{ + "m.relates_to": map[string]any{"rel_type": "m.annotation", "event_id": root, "key": "👍"}, + }) + ts.sendMessage(alice, roomID, "unrelated") + + all := ts.relations(alice, roomID, url.PathEscape(root), nil) + if got := idsOf(all.Chunk); !equalStrings(got, []string{reaction, r2, r1}) { + t.Fatalf("relations = %v, want the reaction and the two replies newest first", got) + } + + threads := ts.relations(alice, roomID, url.PathEscape(root)+"/m.thread", nil) + if got := idsOf(threads.Chunk); !equalStrings(got, []string{r2, r1}) { + t.Errorf("m.thread relations = %v, want the two replies", got) + } + reactions := ts.relations(alice, roomID, url.PathEscape(root)+"/m.annotation/m.reaction", nil) + if got := idsOf(reactions.Chunk); !equalStrings(got, []string{reaction}) { + t.Errorf("m.annotation m.reaction relations = %v, want the reaction", got) + } + + // Paging one at a time walks every child and stops. + var paged []string + from := "" + for range 10 { + query := url.Values{"limit": {"1"}} + if from != "" { + query.Set("from", from) + } + page := ts.relations(alice, roomID, url.PathEscape(root), query) + paged = append(paged, idsOf(page.Chunk)...) + if page.NextBatch == "" { + break + } + from = page.NextBatch + } + if !equalStrings(paged, []string{reaction, r2, r1}) { + t.Errorf("paged relations = %v", paged) + } +} + +func TestRelationsRecurse(t *testing.T) { + ts := newTestServer(t, nil) + alice := ts.register("alice", "correct-horse-battery-staple") + roomID := ts.createRoom(alice, nil) + + root := ts.sendMessage(alice, roomID, "root") + reply := ts.threadReply(alice, roomID, root, "reply") + edit := ts.sendEvent(alice, roomID, "m.room.message", map[string]any{ + "msgtype": "m.text", "body": "* reply!", + "m.new_content": map[string]any{"msgtype": "m.text", "body": "reply!"}, + "m.relates_to": map[string]any{"rel_type": "m.replace", "event_id": reply}, + }) + + flat := ts.relations(alice, roomID, url.PathEscape(root), nil) + if got := idsOf(flat.Chunk); !equalStrings(got, []string{reply}) { + t.Errorf("non-recursive relations = %v, want only the reply", got) + } + deep := ts.relations(alice, roomID, url.PathEscape(root), url.Values{"recurse": {"true"}}) + if got := idsOf(deep.Chunk); !equalStrings(got, []string{edit, reply}) { + t.Errorf("recursive relations = %v, want the edit of the reply too", got) + } + if deep.RecursionDepth != 2 { + t.Errorf("recursion_depth = %d, want 2", deep.RecursionDepth) + } +} + +func TestBundledAggregations(t *testing.T) { + ts := newTestServer(t, nil) + alice := ts.register("alice", "correct-horse-battery-staple") + bob := ts.register("bob", "correct-horse-battery-staple") + roomID := ts.createRoom(alice, map[string]any{"preset": "public_chat"}) + escaped := url.PathEscape(roomID) + expectStatus(t, ts.post("/_matrix/client/v3/join/"+escaped, bob.AccessToken, map[string]any{}), http.StatusOK) + + root := ts.sendMessage(alice, roomID, "root") + ts.threadReply(bob, roomID, root, "first") + latest := ts.threadReply(bob, roomID, root, "second") + edit := ts.sendEvent(alice, roomID, "m.room.message", map[string]any{ + "msgtype": "m.text", "body": "* root, edited", + "m.new_content": map[string]any{"msgtype": "m.text", "body": "root, edited"}, + "m.relates_to": map[string]any{"rel_type": "m.replace", "event_id": root}, + }) + // An edit by somebody else is not an edit. + ts.sendEvent(bob, roomID, "m.room.message", map[string]any{ + "msgtype": "m.text", "body": "* hijacked", + "m.new_content": map[string]any{"msgtype": "m.text", "body": "hijacked"}, + "m.relates_to": map[string]any{"rel_type": "m.replace", "event_id": root}, + }) + + check := func(who string, acc account, participated bool) { + t.Helper() + rec := ts.get("/_matrix/client/v3/rooms/"+escaped+"/event/"+url.PathEscape(root), acc.AccessToken) + expectStatus(t, rec, http.StatusOK) + event := jsonBody(t, rec) + summary := threadSummaryOf(t, event) + if summary == nil { + t.Fatalf("%s: the root has no m.thread summary: %v", who, event["unsigned"]) + } + if summary["count"] != float64(2) { + t.Errorf("%s: thread count = %v, want 2", who, summary["count"]) + } + latestEvent, _ := summary["latest_event"].(map[string]any) + if latestEvent["event_id"] != latest { + t.Errorf("%s: latest_event = %v, want %s", who, latestEvent["event_id"], latest) + } + if summary["current_user_participated"] != participated { + t.Errorf("%s: current_user_participated = %v, want %v", who, summary["current_user_participated"], participated) + } + unsigned, _ := event["unsigned"].(map[string]any) + relations, _ := unsigned["m.relations"].(map[string]any) + replace, _ := relations["m.replace"].(map[string]any) + if replace["event_id"] != edit { + t.Errorf("%s: m.replace = %v, want alice's own edit %s", who, replace["event_id"], edit) + } + } + // Alice started the thread and Bob replied in it: both took part. + check("alice", alice, true) + check("bob", bob, true) + + carol := ts.register("carol", "correct-horse-battery-staple") + expectStatus(t, ts.post("/_matrix/client/v3/join/"+escaped, carol.AccessToken, map[string]any{}), http.StatusOK) + check("carol", carol, false) + + // The same summary rides along on /messages. + page := ts.get("/_matrix/client/v3/rooms/"+escaped+"/messages?dir=b&limit=50", alice.AccessToken) + expectStatus(t, page, http.StatusOK) + chunk, _ := jsonBody(t, page)["chunk"].([]any) + found := false + for _, raw := range chunk { + e, _ := raw.(map[string]any) + if e["event_id"] == root { + found = threadSummaryOf(t, e) != nil + } + } + if !found { + t.Error("the root in /messages carries no m.thread summary") + } +} + +func TestThreadsEndpoint(t *testing.T) { + ts := newTestServer(t, nil) + alice := ts.register("alice", "correct-horse-battery-staple") + bob := ts.register("bob", "correct-horse-battery-staple") + roomID := ts.createRoom(alice, map[string]any{"preset": "public_chat"}) + escaped := url.PathEscape(roomID) + expectStatus(t, ts.post("/_matrix/client/v3/join/"+escaped, bob.AccessToken, map[string]any{}), http.StatusOK) + + first := ts.sendMessage(alice, roomID, "first root") + second := ts.sendMessage(bob, roomID, "second root") + ts.sendMessage(alice, roomID, "not a thread") + ts.threadReply(alice, roomID, first, "in the first") + ts.threadReply(alice, roomID, second, "in the second") + ts.threadReply(alice, roomID, first, "the first again") + + threads := func(acc account, query string) relationsResult { + t.Helper() + rec := ts.get("/_matrix/client/v1/rooms/"+escaped+"/threads"+query, acc.AccessToken) + expectStatus(t, rec, http.StatusOK) + return decode[relationsResult](t, rec) + } + + all := threads(bob, "") + if got := idsOf(all.Chunk); !equalStrings(got, []string{first, second}) { + t.Fatalf("threads = %v, want the first (latest activity) then the second", got) + } + + // Bob started the second thread and never replied in the first. + mine := threads(bob, "?include=participated") + if got := idsOf(mine.Chunk); !equalStrings(got, []string{second}) { + t.Errorf("participated threads = %v, want only bob's own", got) + } + + page1 := threads(bob, "?limit=1") + if got := idsOf(page1.Chunk); !equalStrings(got, []string{first}) || page1.NextBatch == "" { + t.Fatalf("first page = %v, next_batch %q", got, page1.NextBatch) + } + page2 := threads(bob, "?limit=1&from="+url.QueryEscape(page1.NextBatch)) + if got := idsOf(page2.Chunk); !equalStrings(got, []string{second}) || page2.NextBatch != "" { + t.Errorf("second page = %v, next_batch %q", got, page2.NextBatch) + } + + rec := ts.get("/_matrix/client/v1/rooms/"+escaped+"/threads?include=everything", bob.AccessToken) + expectStatus(t, rec, http.StatusBadRequest) +} + +func TestRedactedRelationDisappears(t *testing.T) { + ts := newTestServer(t, nil) + alice := ts.register("alice", "correct-horse-battery-staple") + roomID := ts.createRoom(alice, nil) + escaped := url.PathEscape(roomID) + + root := ts.sendMessage(alice, roomID, "root") + reply := ts.threadReply(alice, roomID, root, "reply") + expectStatus(t, ts.put("/_matrix/client/v3/rooms/"+escaped+"/redact/"+url.PathEscape(reply)+"/r1", + alice.AccessToken, map[string]any{}), http.StatusOK) + + if got := ts.relations(alice, roomID, url.PathEscape(root), nil); len(got.Chunk) != 0 { + t.Errorf("relations after the reply was redacted = %v", idsOf(got.Chunk)) + } + rec := ts.get("/_matrix/client/v3/rooms/"+escaped+"/event/"+url.PathEscape(root), alice.AccessToken) + if summary := threadSummaryOf(t, jsonBody(t, rec)); summary != nil { + t.Errorf("the root still has a thread summary: %v", summary) + } +} + +func TestRelationsOfAnInvisibleEvent(t *testing.T) { + ts := newTestServer(t, nil) + alice := ts.register("alice", "correct-horse-battery-staple") + bob := ts.register("bob", "correct-horse-battery-staple") + roomID := ts.createRoom(alice, nil) + root := ts.sendMessage(alice, roomID, "secret") + + rec := ts.get("/_matrix/client/v1/rooms/"+url.PathEscape(roomID)+"/relations/"+url.PathEscape(root), + bob.AccessToken) + expectStatus(t, rec, http.StatusForbidden) +} + +func idsOf(events []syncEvent) []string { + out := make([]string, 0, len(events)) + for _, e := range events { + out = append(out, e.EventID) + } + return out +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/server/internal/clientapi/search.go b/server/internal/clientapi/search.go new file mode 100644 index 0000000..e4b5d7c --- /dev/null +++ b/server/internal/clientapi/search.go @@ -0,0 +1,27 @@ +package clientapi + +import ( + "net/http" + + "github.com/dariuszjaniak/warp/server/internal/httputil" + "github.com/dariuszjaniak/warp/server/internal/syncapi" + "github.com/dariuszjaniak/warp/server/internal/userapi" +) + +// handleSearch implements POST /_matrix/client/v3/search. +func (c *ClientAPI) handleSearch(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + var body syncapi.SearchBody + if err := httputil.DecodeJSON(w, r, &body); err != nil { + return err + } + resp, err := c.sync.Search(r.Context(), syncapi.SearchRequest{ + User: auth.UserID, + NextBatch: r.URL.Query().Get("next_batch"), + Body: body, + }) + if err != nil { + return syncError(err) + } + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} diff --git a/server/internal/clientapi/search_test.go b/server/internal/clientapi/search_test.go new file mode 100644 index 0000000..c4f4b4c --- /dev/null +++ b/server/internal/clientapi/search_test.go @@ -0,0 +1,147 @@ +package clientapi_test + +import ( + "net/http" + "net/url" + "testing" +) + +type searchResult struct { + SearchCategories struct { + RoomEvents struct { + Count int `json:"count"` + Highlights []string `json:"highlights"` + NextBatch string `json:"next_batch"` + Results []struct { + Rank float64 `json:"rank"` + Result syncEvent `json:"result"` + Context *struct { + EventsBefore []syncEvent `json:"events_before"` + EventsAfter []syncEvent `json:"events_after"` + ProfileInfo map[string]map[string]any `json:"profile_info"` + } `json:"context"` + } `json:"results"` + State map[string][]syncEvent `json:"state"` + Groups map[string]map[string]map[string]any `json:"groups"` + } `json:"room_events"` + } `json:"search_categories"` +} + +func (ts *testServer) search(acc account, criteria map[string]any, nextBatch string) searchResult { + ts.t.Helper() + path := "/_matrix/client/v3/search" + if nextBatch != "" { + path += "?next_batch=" + url.QueryEscape(nextBatch) + } + rec := ts.post(path, acc.AccessToken, map[string]any{ + "search_categories": map[string]any{"room_events": criteria}, + }) + expectStatus(ts.t, rec, http.StatusOK) + return decode[searchResult](ts.t, rec) +} + +func TestSearchFindsOnlyWhatTheUserMaySee(t *testing.T) { + ts := newTestServer(t, nil) + alice := ts.register("alice", "correct-horse-battery-staple") + bob := ts.register("bob", "correct-horse-battery-staple") + + shared := ts.createRoom(alice, map[string]any{"preset": "public_chat"}) + private := ts.createRoom(alice, nil) + expectStatus(t, ts.post("/_matrix/client/v3/join/"+url.PathEscape(shared), bob.AccessToken, map[string]any{}), http.StatusOK) + + ts.sendMessage(alice, shared, "the quick brown fox") + ts.sendMessage(alice, private, "a secret fox") + ts.sendMessage(bob, shared, "lazy dog") + + got := ts.search(bob, map[string]any{"search_term": "fox"}, "").SearchCategories.RoomEvents + if got.Count != 1 || len(got.Results) != 1 || got.Results[0].Result.Content["body"] != "the quick brown fox" { + t.Fatalf("bob's search for fox = %+v, want the one message in the room he is in", got) + } + if len(got.Highlights) != 1 || got.Highlights[0] != "fox" { + t.Errorf("highlights = %v", got.Highlights) + } + + all := ts.search(alice, map[string]any{"search_term": "fox"}, "").SearchCategories.RoomEvents + if all.Count != 2 { + t.Errorf("alice's search for fox found %d, want both rooms' messages", all.Count) + } + only := ts.search(alice, map[string]any{ + "search_term": "fox", + "filter": map[string]any{"rooms": []string{private}}, + }, "").SearchCategories.RoomEvents + if only.Count != 1 || only.Results[0].Result.RoomID != private { + t.Errorf("search filtered to one room = %+v", only) + } + + // After Bob leaves, what was said while he was there stays findable + // and what comes later does not. + expectStatus(t, ts.post("/_matrix/client/v3/rooms/"+url.PathEscape(shared)+"/leave", bob.AccessToken, map[string]any{}), http.StatusOK) + ts.sendMessage(alice, shared, "another fox, later") + after := ts.search(bob, map[string]any{"search_term": "fox"}, "").SearchCategories.RoomEvents + if after.Count != 1 { + t.Errorf("after leaving, bob's search found %d, want only the message from before", after.Count) + } +} + +func TestSearchContextStateGroupsAndPaging(t *testing.T) { + ts := newTestServer(t, nil) + alice := ts.register("alice", "correct-horse-battery-staple") + roomID := ts.createRoom(alice, nil) + + ts.sendMessage(alice, roomID, "before") + ts.sendMessage(alice, roomID, "needle one") + ts.sendMessage(alice, roomID, "after") + ts.sendMessage(alice, roomID, "needle two") + ts.sendMessage(alice, roomID, "needle three") + + got := ts.search(alice, map[string]any{ + "search_term": "needle", + "order_by": "recent", + "event_context": map[string]any{"before_limit": 1, "after_limit": 1, "include_profile": true}, + "include_state": true, + "groupings": map[string]any{"group_by": []map[string]any{{"key": "room_id"}}}, + "filter": map[string]any{"limit": 2}, + }, "").SearchCategories.RoomEvents + + if got.Count != 3 || len(got.Results) != 2 { + t.Fatalf("first page: count %d, %d results", got.Count, len(got.Results)) + } + if body := got.Results[0].Result.Content["body"]; body != "needle three" { + t.Errorf("newest first: got %v", body) + } + ctx := got.Results[1].Context + if ctx == nil || len(ctx.EventsBefore) != 1 || len(ctx.EventsAfter) != 1 || + ctx.EventsBefore[0].Content["body"] != "after" || ctx.EventsAfter[0].Content["body"] != "needle three" { + t.Errorf("context of needle two = %+v", ctx) + } + if _, ok := ctx.ProfileInfo[alice.UserID]; !ok { + t.Errorf("profile_info has no entry for alice: %v", ctx.ProfileInfo) + } + if len(got.State[roomID]) == 0 { + t.Error("include_state returned no state for the room") + } + if group := got.Groups["room_id"][roomID]; group == nil { + t.Errorf("groups = %v", got.Groups) + } + if got.NextBatch == "" { + t.Fatal("a full first page carried no next_batch") + } + + next := ts.search(alice, map[string]any{ + "search_term": "needle", "order_by": "recent", "filter": map[string]any{"limit": 2}, + }, got.NextBatch).SearchCategories.RoomEvents + if len(next.Results) != 1 || next.Results[0].Result.Content["body"] != "needle one" { + t.Errorf("second page = %+v", next.Results) + } + + ranked := ts.search(alice, map[string]any{"search_term": "needle", "filter": map[string]any{"limit": 2}}, "").SearchCategories.RoomEvents + rest := ts.search(alice, map[string]any{"search_term": "needle", "filter": map[string]any{"limit": 2}}, ranked.NextBatch).SearchCategories.RoomEvents + if len(ranked.Results)+len(rest.Results) != 3 || rest.NextBatch != "" { + t.Errorf("ranked pages: %d then %d results, next %q", len(ranked.Results), len(rest.Results), rest.NextBatch) + } + + rec := ts.post("/_matrix/client/v3/search", alice.AccessToken, map[string]any{ + "search_categories": map[string]any{"room_events": map[string]any{"search_term": "x", "order_by": "loudest"}}, + }) + expectStatus(t, rec, http.StatusBadRequest) +} diff --git a/server/internal/clientapi/sliding.go b/server/internal/clientapi/sliding.go new file mode 100644 index 0000000..459b3f4 --- /dev/null +++ b/server/internal/clientapi/sliding.go @@ -0,0 +1,47 @@ +package clientapi + +import ( + "errors" + "net/http" + + "github.com/dariuszjaniak/warp/server/internal/httputil" + "github.com/dariuszjaniak/warp/server/internal/syncapi" + "github.com/dariuszjaniak/warp/server/internal/userapi" +) + +// slidingSyncPath is MSC4186's endpoint. It is still unstable: the MSC has +// not been merged, and the prefix is what every client implementing it +// sends. +const slidingSyncPath = "/_matrix/client/unstable/org.matrix.simplified_msc3575/sync" + +// handleSlidingSync implements POST slidingSyncPath. +func (c *ClientAPI) handleSlidingSync(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + query := r.URL.Query() + timeout, err := durationParam(query.Get("timeout")) + if err != nil { + return err + } + var body syncapi.SlidingBody + if _, err := httputil.DecodeOptionalJSON(w, r, &body); err != nil { + return err + } + + resp, err := c.sync.SlidingSync(r.Context(), syncapi.SlidingRequest{ + User: auth.UserID, + Localpart: auth.Localpart, + Device: auth.DeviceID, + Pos: query.Get("pos"), + Timeout: timeout, + SetPresence: query.Get("set_presence"), + Body: body, + }) + if err != nil { + if ctxErr := r.Context().Err(); ctxErr != nil && errors.Is(err, ctxErr) { + return nil //nolint:nilerr // the client is gone; there is nobody to answer + } + return syncError(err) + } + httputil.NoCache(w) + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} diff --git a/server/internal/clientapi/sliding_test.go b/server/internal/clientapi/sliding_test.go new file mode 100644 index 0000000..c18a91b --- /dev/null +++ b/server/internal/clientapi/sliding_test.go @@ -0,0 +1,332 @@ +package clientapi_test + +import ( + "fmt" + "net/http" + "net/url" + "testing" + "time" + + "github.com/dariuszjaniak/warp/server/internal/httputil" +) + +// Simplified Sliding Sync (MSC4186). + +const slidingPath = "/_matrix/client/unstable/org.matrix.simplified_msc3575/sync" + +type slidingResult struct { + Pos string `json:"pos"` + Lists map[string]struct { + Count int `json:"count"` + } `json:"lists"` + Rooms map[string]struct { + Name string `json:"name"` + Initial bool `json:"initial"` + RequiredState []syncEvent `json:"required_state"` + Timeline []syncEvent `json:"timeline"` + PrevBatch string `json:"prev_batch"` + Limited bool `json:"limited"` + NumLive *int `json:"num_live"` + BumpStamp int64 `json:"bump_stamp"` + JoinedCount *int `json:"joined_count"` + InviteState []map[string]any `json:"invite_state"` + Heroes []map[string]any `json:"heroes"` + } `json:"rooms"` + Extensions struct { + ToDevice *struct { + NextBatch string `json:"next_batch"` + Events []map[string]any `json:"events"` + } `json:"to_device"` + E2EE *struct { + DeviceLists struct { + Changed []string `json:"changed"` + } `json:"device_lists"` + DeviceOneTimeKeysCount map[string]int `json:"device_one_time_keys_count"` + } `json:"e2ee"` + AccountData *struct { + Global []map[string]any `json:"global"` + Rooms map[string][]map[string]any `json:"rooms"` + } `json:"account_data"` + Receipts *struct { + Rooms map[string]map[string]any `json:"rooms"` + } `json:"receipts"` + Typing *struct { + Rooms map[string]map[string]any `json:"rooms"` + } `json:"typing"` + } `json:"extensions"` +} + +func (ts *testServer) slidingRaw(acc account, pos string, timeout time.Duration, body map[string]any) (int, slidingResult, map[string]any) { + ts.t.Helper() + query := url.Values{} + if pos != "" { + query.Set("pos", pos) + } + query.Set("timeout", fmt.Sprint(timeout.Milliseconds())) + rec := ts.post(slidingPath+"?"+query.Encode(), acc.AccessToken, body) + if rec.Code != http.StatusOK { + return rec.Code, slidingResult{}, jsonBody(ts.t, rec) + } + return rec.Code, decode[slidingResult](ts.t, rec), nil +} + +func (ts *testServer) sliding(acc account, pos string, body map[string]any) slidingResult { + ts.t.Helper() + code, result, errBody := ts.slidingRaw(acc, pos, 0, body) + if code != http.StatusOK { + ts.t.Fatalf("sliding sync answered %d: %v", code, errBody) + } + if result.Pos == "" { + ts.t.Fatal("sliding sync returned no pos") + } + return result +} + +func roomList(ranges [][2]int, timelineLimit int, requiredState [][]string) map[string]any { + return map[string]any{ + "lists": map[string]any{ + "all": map[string]any{ + "ranges": ranges, + "timeline_limit": timelineLimit, + "required_state": requiredState, + }, + }, + } +} + +func TestSlidingSyncListWindowAndIncrementals(t *testing.T) { + ts := newTestServer(t, nil) + alice := ts.register("alice", "correct-horse-battery-staple") + + oldest := ts.createRoom(alice, map[string]any{"name": "Oldest"}) + middle := ts.createRoom(alice, map[string]any{"name": "Middle"}) + newest := ts.createRoom(alice, map[string]any{"name": "Newest"}) + ts.sendMessage(alice, oldest, "o1") + ts.sendMessage(alice, middle, "m1") + ts.sendMessage(alice, newest, "n1") + ts.sendMessage(alice, newest, "n2") + + body := roomList([][2]int{{0, 1}}, 2, [][]string{{"m.room.name", ""}}) + first := ts.sliding(alice, "", body) + if first.Lists["all"].Count != 3 { + t.Fatalf("list count = %d, want 3", first.Lists["all"].Count) + } + if len(first.Rooms) != 2 { + t.Fatalf("rooms = %v, want the two most recently active", keysOf(first.Rooms)) + } + room, ok := first.Rooms[newest] + if !ok || !room.Initial || room.Name != "Newest" { + t.Fatalf("newest room = %+v", room) + } + if got := bodiesOf(room.Timeline); len(got) != 2 || got[1] != "n2" || !room.Limited || room.PrevBatch == "" { + t.Errorf("newest timeline = %v (limited %v, prev_batch %q)", got, room.Limited, room.PrevBatch) + } + if len(room.RequiredState) != 1 || room.RequiredState[0].Type != "m.room.name" { + t.Errorf("required_state = %+v", room.RequiredState) + } + if room.JoinedCount == nil || *room.JoinedCount != 1 || room.BumpStamp == 0 { + t.Errorf("counts and bump stamp = %v %d", room.JoinedCount, room.BumpStamp) + } + if _, ok := first.Rooms[middle]; !ok { + t.Errorf("middle room missing from the window: %v", keysOf(first.Rooms)) + } + + // Nothing has happened: nothing is sent. + idle := ts.sliding(alice, first.Pos, body) + if len(idle.Rooms) != 0 { + t.Errorf("an idle incremental sent %v", keysOf(idle.Rooms)) + } + + // A message in a room already sent comes as an increment. + ts.sendMessage(alice, middle, "m2") + inc := ts.sliding(alice, idle.Pos, body) + got, ok := inc.Rooms[middle] + if !ok || got.Initial || len(got.Timeline) != 1 || got.Timeline[0].Content["body"] != "m2" { + t.Fatalf("increment for middle = %+v (rooms %v)", got, keysOf(inc.Rooms)) + } + if got.NumLive == nil || *got.NumLive != 1 { + t.Errorf("num_live = %v, want 1", got.NumLive) + } + + // The oldest room becomes active, moves into the window, and is sent + // in full because this connection has never seen it. + ts.sendMessage(alice, oldest, "o2") + bumped := ts.sliding(alice, inc.Pos, body) + if o, ok := bumped.Rooms[oldest]; !ok || !o.Initial { + t.Errorf("the bumped room = %+v (rooms %v)", o, keysOf(bumped.Rooms)) + } + + // A retry of an earlier pos still works; a made-up one does not. + if retry := ts.sliding(alice, inc.Pos, body); retry.Pos == "" { + t.Error("retrying the previous pos failed") + } + code, _, errBody := ts.slidingRaw(alice, "nonsense.5", 0, body) + if code != http.StatusBadRequest || errBody["errcode"] != httputil.ErrCodeUnknownPos { + t.Errorf("an unknown pos answered %d %v, want M_UNKNOWN_POS", code, errBody) + } +} + +func TestSlidingSyncInvitesSubscriptionsAndLeaves(t *testing.T) { + ts := newTestServer(t, nil) + alice := ts.register("alice", "correct-horse-battery-staple") + bob := ts.register("bob", "correct-horse-battery-staple") + + joined := ts.createRoom(bob, nil) + ts.sendMessage(bob, joined, "hello") + invited := ts.createRoom(alice, map[string]any{"name": "Party", "invite": []string{bob.UserID}}) + + body := map[string]any{ + "lists": map[string]any{ + "invites": map[string]any{"ranges": [][2]int{{0, 10}}, "filters": map[string]any{"is_invite": true}}, + "rooms": map[string]any{ + "ranges": [][2]int{{0, 10}}, "filters": map[string]any{"is_invite": false}, + "timeline_limit": 5, "required_state": [][]string{{"m.room.member", "$LAZY"}}, + }, + }, + } + first := ts.sliding(bob, "", body) + if first.Lists["invites"].Count != 1 || first.Lists["rooms"].Count != 1 { + t.Fatalf("list counts = %+v", first.Lists) + } + if inv := first.Rooms[invited]; len(inv.InviteState) == 0 || inv.Timeline != nil { + t.Errorf("the invite = %+v, want stripped state and no timeline", inv) + } + lazy := first.Rooms[joined].RequiredState + if len(lazy) != 1 || lazy[0].Type != "m.room.member" || *lazy[0].StateKey != bob.UserID { + t.Errorf("lazy-loaded members = %+v, want only bob, who sent the timeline", lazy) + } + + // A subscription to a room outside every list. + public := ts.createRoom(alice, map[string]any{"name": "Public", "preset": "public_chat"}) + expectStatus(t, ts.post("/_matrix/client/v3/join/"+url.PathEscape(public), bob.AccessToken, map[string]any{}), http.StatusOK) + sub := ts.sliding(bob, first.Pos, map[string]any{ + "room_subscriptions": map[string]any{public: map[string]any{"timeline_limit": 1, "required_state": [][]string{{"*", "*"}}}}, + }) + if r, ok := sub.Rooms[public]; !ok || !r.Initial || len(r.RequiredState) < 5 { + t.Fatalf("subscribed room = %+v", r) + } + + // Joining the invited room turns it into a room sent in full. + expectStatus(t, ts.post("/_matrix/client/v3/join/"+url.PathEscape(invited), bob.AccessToken, map[string]any{}), http.StatusOK) + afterJoin := ts.sliding(bob, sub.Pos, body) + if r, ok := afterJoin.Rooms[invited]; !ok || !r.Initial || len(r.Timeline) == 0 { + t.Errorf("the room just joined = %+v", r) + } + + // Leaving is reported, with the leave in the timeline. + expectStatus(t, ts.post("/_matrix/client/v3/rooms/"+url.PathEscape(invited)+"/leave", bob.AccessToken, map[string]any{}), http.StatusOK) + afterLeave := ts.sliding(bob, afterJoin.Pos, body) + r, ok := afterLeave.Rooms[invited] + if !ok || len(r.Timeline) == 0 || r.Timeline[len(r.Timeline)-1].Type != "m.room.member" { + t.Errorf("the room just left = %+v (rooms %v)", r, keysOf(afterLeave.Rooms)) + } +} + +func TestSlidingSyncExtensions(t *testing.T) { + ts := newTestServer(t, nil) + alice := ts.register("alice", "correct-horse-battery-staple") + roomID := ts.createRoom(alice, nil) + eventID := ts.sendMessage(alice, roomID, "read me") + + body := roomList([][2]int{{0, 0}}, 1, nil) + body["extensions"] = map[string]any{ + "to_device": map[string]any{"enabled": true}, + "e2ee": map[string]any{"enabled": true}, + "account_data": map[string]any{"enabled": true}, + "receipts": map[string]any{"enabled": true}, + "typing": map[string]any{"enabled": true}, + } + + expectStatus(t, ts.put("/_matrix/client/v3/user/"+url.PathEscape(alice.UserID)+"/account_data/com.example.setting", + alice.AccessToken, map[string]any{"on": true}), http.StatusOK) + expectStatus(t, ts.post(fmt.Sprintf("/_matrix/client/v3/rooms/%s/receipt/m.read/%s", + url.PathEscape(roomID), url.PathEscape(eventID)), alice.AccessToken, map[string]any{}), http.StatusOK) + expectStatus(t, ts.put("/_matrix/client/v3/sendToDevice/m.test/t1", alice.AccessToken, map[string]any{ + "messages": map[string]any{alice.UserID: map[string]any{alice.DeviceID: map[string]any{"hello": "device"}}}, + }), http.StatusOK) + + first := ts.sliding(alice, "", body) + ext := first.Extensions + if ext.ToDevice == nil || len(ext.ToDevice.Events) != 1 || ext.ToDevice.NextBatch == "" { + t.Fatalf("to_device = %+v", ext.ToDevice) + } + if ext.E2EE == nil || ext.E2EE.DeviceOneTimeKeysCount == nil { + t.Errorf("e2ee = %+v", ext.E2EE) + } + foundSetting := false + if ext.AccountData != nil { + for _, e := range ext.AccountData.Global { + if e["type"] == "com.example.setting" { + foundSetting = true + } + } + } + if !foundSetting { + t.Errorf("account_data = %+v", ext.AccountData) + } + if ext.Receipts == nil || ext.Receipts.Rooms[roomID] == nil { + t.Errorf("receipts = %+v", ext.Receipts) + } + + // Coming back with the to-device token acknowledges the batch. + body["extensions"].(map[string]any)["to_device"] = map[string]any{"enabled": true, "since": ext.ToDevice.NextBatch} + second := ts.sliding(alice, first.Pos, body) + if second.Extensions.ToDevice == nil || len(second.Extensions.ToDevice.Events) != 0 { + t.Errorf("an acknowledged batch came again: %+v", second.Extensions.ToDevice) + } + + // Typing is sent when it starts and again when it stops. + expectStatus(t, ts.put(fmt.Sprintf("/_matrix/client/v3/rooms/%s/typing/%s", url.PathEscape(roomID), url.PathEscape(alice.UserID)), + alice.AccessToken, map[string]any{"typing": true, "timeout": 30000}), http.StatusOK) + typing := ts.sliding(alice, second.Pos, body) + if typing.Extensions.Typing == nil || typing.Extensions.Typing.Rooms[roomID] == nil { + t.Fatalf("typing = %+v", typing.Extensions.Typing) + } + again := ts.sliding(alice, typing.Pos, body) + if len(again.Extensions.Typing.Rooms) != 0 { + t.Errorf("unchanged typing was sent again: %+v", again.Extensions.Typing.Rooms) + } + expectStatus(t, ts.put(fmt.Sprintf("/_matrix/client/v3/rooms/%s/typing/%s", url.PathEscape(roomID), url.PathEscape(alice.UserID)), + alice.AccessToken, map[string]any{"typing": false}), http.StatusOK) + stopped := ts.sliding(alice, again.Pos, body) + if stopped.Extensions.Typing.Rooms[roomID] == nil { + t.Errorf("stopping typing was not sent: %+v", stopped.Extensions.Typing.Rooms) + } +} + +func TestSlidingSyncLongPoll(t *testing.T) { + ts := newTestServer(t, nil) + alice := ts.register("alice", "correct-horse-battery-staple") + roomID := ts.createRoom(alice, nil) + body := roomList([][2]int{{0, 0}}, 5, nil) + first := ts.sliding(alice, "", body) + + done := make(chan slidingResult, 1) + start := time.Now() + go func() { + _, result, _ := ts.slidingRaw(alice, first.Pos, 10*time.Second, body) + done <- result + }() + time.Sleep(200 * time.Millisecond) + ts.sendMessage(alice, roomID, "wake up") + + select { + case result := <-done: + if r, ok := result.Rooms[roomID]; !ok || len(r.Timeline) != 1 { + t.Errorf("the woken response = %+v", result.Rooms) + } + if time.Since(start) > 5*time.Second { + t.Errorf("the long poll took %s to wake", time.Since(start)) + } + case <-time.After(15 * time.Second): + t.Fatal("the long poll never returned") + } +} + +func keysOf[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/server/internal/clientapi/spaces.go b/server/internal/clientapi/spaces.go new file mode 100644 index 0000000..76a0551 --- /dev/null +++ b/server/internal/clientapi/spaces.go @@ -0,0 +1,105 @@ +package clientapi + +import ( + "net/http" + "strconv" + "strings" + + "github.com/dariuszjaniak/warp/server/internal/httputil" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/userapi" +) + +// hierarchyResponse is what GET /hierarchy returns. +type hierarchyResponse struct { + Rooms []roomserver.SpaceRoom `json:"rooms"` + NextBatch string `json:"next_batch,omitempty"` +} + +// hierarchyTokenPrefix marks a /hierarchy token: how far into the walk +// the next page starts. +const hierarchyTokenPrefix = "h" + +// handleHierarchy implements GET /_matrix/client/v1/rooms/{roomID}/hierarchy. +func (c *ClientAPI) handleHierarchy(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + roomID, err := pathRoomID(r) + if err != nil { + return err + } + query := r.URL.Query() + + limit, err := intParam(query.Get("limit")) + if err != nil { + return err + } + suggestedOnly, err := boolParam(query.Get("suggested_only")) + if err != nil { + return err + } + maxDepth := -1 + if raw := query.Get("max_depth"); raw != "" { + if maxDepth, err = strconv.Atoi(raw); err != nil || maxDepth < 0 { + return httputil.InvalidParam("max_depth must be a non-negative integer") + } + } + from := 0 + if raw := query.Get("from"); raw != "" { + body, ok := strings.CutPrefix(raw, hierarchyTokenPrefix) + if from, err = strconv.Atoi(body); !ok || err != nil || from < 0 { + return httputil.InvalidParam("%q is not a hierarchy token", raw) + } + } + + page, err := c.rooms.Hierarchy(r.Context(), roomserver.HierarchyRequest{ + RoomID: roomID, + User: auth.UserID, + SuggestedOnly: suggestedOnly, + MaxDepth: maxDepth, + From: from, + Limit: limit, + }) + if err != nil { + return roomError(err) + } + resp := hierarchyResponse{Rooms: page.Rooms} + if resp.Rooms == nil { + resp.Rooms = []roomserver.SpaceRoom{} + } + if page.Next != 0 { + resp.NextBatch = hierarchyTokenPrefix + strconv.Itoa(page.Next) + } + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} + +// handleRoomSummary implements +// GET /_matrix/client/v1/room_summary/{roomIDOrAlias}. +func (c *ClientAPI) handleRoomSummary(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error { + target := r.PathValue("roomIDOrAlias") + via := r.URL.Query()["via"] + + roomID := target + switch { + case strings.HasPrefix(target, "#"): + resolved, err := c.resolveAlias(r.Context(), target) + if err != nil { + return roomError(err) + } + roomID = resolved.RoomID + via = append(via, resolved.Servers...) + case id.ValidateRoomID(target) != nil: + return httputil.InvalidParam("%s is not a room ID or alias", target) + } + + resp, err := c.rooms.RoomSummary(r.Context(), roomserver.RoomSummaryRequest{ + RoomID: roomID, + User: auth.UserID, + Via: via, + }) + if err != nil { + return roomError(err) + } + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} diff --git a/server/internal/clientapi/spaces_test.go b/server/internal/clientapi/spaces_test.go new file mode 100644 index 0000000..b4e99ae --- /dev/null +++ b/server/internal/clientapi/spaces_test.go @@ -0,0 +1,182 @@ +package clientapi_test + +import ( + "net/http" + "net/url" + "testing" +) + +type hierarchyResult struct { + Rooms []struct { + RoomID string `json:"room_id"` + Name string `json:"name"` + RoomType string `json:"room_type"` + JoinRule string `json:"join_rule"` + NumJoinedMembers int `json:"num_joined_members"` + ChildrenState []map[string]any `json:"children_state"` + } `json:"rooms"` + NextBatch string `json:"next_batch"` +} + +func (h hierarchyResult) ids() []string { + out := make([]string, 0, len(h.Rooms)) + for _, r := range h.Rooms { + out = append(out, r.RoomID) + } + return out +} + +func (ts *testServer) hierarchy(acc account, roomID, query string) hierarchyResult { + ts.t.Helper() + rec := ts.get("/_matrix/client/v1/rooms/"+url.PathEscape(roomID)+"/hierarchy"+query, acc.AccessToken) + expectStatus(ts.t, rec, http.StatusOK) + return decode[hierarchyResult](ts.t, rec) +} + +// addChild puts a room into a space. +func (ts *testServer) addChild(acc account, spaceID, childID string, content map[string]any) { + ts.t.Helper() + if content == nil { + content = map[string]any{} + } + if _, ok := content["via"]; !ok { + content["via"] = []string{"warp.example"} + } + rec := ts.put("/_matrix/client/v3/rooms/"+url.PathEscape(spaceID)+"/state/m.space.child/"+url.PathEscape(childID), + acc.AccessToken, content) + expectStatus(ts.t, rec, http.StatusOK) +} + +func (ts *testServer) createSpace(acc account, name string, public bool) string { + ts.t.Helper() + body := map[string]any{"name": name, "creation_content": map[string]any{"type": "m.space"}} + if public { + body["preset"] = "public_chat" + } + return ts.createRoom(acc, body) +} + +func TestSpaceHierarchy(t *testing.T) { + ts := newTestServer(t, nil) + alice := ts.register("alice", "correct-horse-battery-staple") + bob := ts.register("bob", "correct-horse-battery-staple") + + space := ts.createSpace(alice, "Space", true) + sub := ts.createSpace(alice, "Subspace", true) + general := ts.createRoom(alice, map[string]any{"name": "General", "preset": "public_chat"}) + random := ts.createRoom(alice, map[string]any{"name": "Random", "preset": "public_chat"}) + secret := ts.createRoom(alice, map[string]any{"name": "Secret"}) + deep := ts.createRoom(alice, map[string]any{"name": "Deep", "preset": "public_chat"}) + + ts.addChild(alice, space, random, map[string]any{"order": "b"}) + ts.addChild(alice, space, general, map[string]any{"order": "a", "suggested": true}) + ts.addChild(alice, space, sub, nil) + ts.addChild(alice, space, secret, nil) + ts.addChild(alice, sub, deep, nil) + // A child event with no via is a removed child. + removed := ts.createRoom(alice, map[string]any{"name": "Gone", "preset": "public_chat"}) + ts.addChild(alice, space, removed, map[string]any{"via": []string{}}) + + // Alice is in every room, so she sees all of them: the root, its + // children by order and then by timestamp, then the subspace's child. + all := ts.hierarchy(alice, space, "") + want := []string{space, general, random, sub, secret, deep} + if got := all.ids(); !equalStrings(got, want) { + t.Fatalf("alice's hierarchy = %v, want %v", got, want) + } + if all.Rooms[0].RoomType != "m.space" || len(all.Rooms[0].ChildrenState) != 4 { + t.Errorf("root = %+v, want a space with four live children", all.Rooms[0]) + } + + // Bob is in nothing: the invite-only room is hidden from him. + if got := ts.hierarchy(bob, space, "").ids(); !equalStrings(got, []string{space, general, random, sub, deep}) { + t.Errorf("bob's hierarchy = %v", got) + } + + if got := ts.hierarchy(alice, space, "?max_depth=1").ids(); !equalStrings(got, []string{space, general, random, sub, secret}) { + t.Errorf("max_depth=1 = %v", got) + } + if got := ts.hierarchy(alice, space, "?suggested_only=true").ids(); !equalStrings(got, []string{space, general}) { + t.Errorf("suggested_only = %v", got) + } + + // Paging two at a time walks the same order. + var paged []string + from := "" + for range 10 { + query := "?limit=2" + if from != "" { + query += "&from=" + url.QueryEscape(from) + } + page := ts.hierarchy(alice, space, query) + paged = append(paged, page.ids()...) + if page.NextBatch == "" { + break + } + from = page.NextBatch + } + if !equalStrings(paged, want) { + t.Errorf("paged hierarchy = %v, want %v", paged, want) + } +} + +func TestSpaceHierarchyAccess(t *testing.T) { + ts := newTestServer(t, nil) + alice := ts.register("alice", "correct-horse-battery-staple") + bob := ts.register("bob", "correct-horse-battery-staple") + carol := ts.register("carol", "correct-horse-battery-staple") + + private := ts.createSpace(alice, "Private", false) + rec := ts.get("/_matrix/client/v1/rooms/"+url.PathEscape(private)+"/hierarchy", bob.AccessToken) + expectStatus(t, rec, http.StatusForbidden) + + // A restricted room is visible to the members of the room it lets in. + space := ts.createSpace(alice, "Space", true) + restricted := ts.createRoom(alice, map[string]any{ + "name": "Members only", + "initial_state": []map[string]any{{ + "type": "m.room.join_rules", "state_key": "", + "content": map[string]any{ + "join_rule": "restricted", + "allow": []map[string]any{{"type": "m.room_membership", "room_id": space}}, + }, + }}, + }) + ts.addChild(alice, space, restricted, nil) + expectStatus(t, ts.post("/_matrix/client/v3/join/"+url.PathEscape(space), bob.AccessToken, map[string]any{}), http.StatusOK) + + if got := ts.hierarchy(bob, space, "").ids(); !equalStrings(got, []string{space, restricted}) { + t.Errorf("a member of the space sees %v, want the restricted room too", got) + } + if got := ts.hierarchy(carol, space, "").ids(); !equalStrings(got, []string{space}) { + t.Errorf("a stranger sees %v, want the space alone", got) + } +} + +func TestRoomSummary(t *testing.T) { + ts := newTestServer(t, nil) + alice := ts.register("alice", "correct-horse-battery-staple") + bob := ts.register("bob", "correct-horse-battery-staple") + public := ts.createRoom(alice, map[string]any{ + "name": "Lobby", "topic": "Say hello", "preset": "public_chat", "room_alias_name": "lobby", + }) + private := ts.createRoom(alice, map[string]any{"name": "Private"}) + + for _, target := range []string{public, "#lobby:warp.example"} { + rec := ts.get("/_matrix/client/v1/room_summary/"+url.PathEscape(target), bob.AccessToken) + expectStatus(t, rec, http.StatusOK) + body := jsonBody(t, rec) + if body["room_id"] != public || body["name"] != "Lobby" || body["topic"] != "Say hello" || + body["membership"] != "leave" || body["join_rule"] != "public" || body["room_version"] == nil { + t.Errorf("summary of %s = %v", target, body) + } + } + + rec := ts.get("/_matrix/client/v1/room_summary/"+url.PathEscape(private), bob.AccessToken) + expectStatus(t, rec, http.StatusForbidden) + rec = ts.get("/_matrix/client/v1/room_summary/"+url.PathEscape(private), alice.AccessToken) + expectStatus(t, rec, http.StatusOK) + if jsonBody(t, rec)["membership"] != "join" { + t.Errorf("alice's summary of her own room says membership %v", jsonBody(t, rec)["membership"]) + } +} diff --git a/server/internal/clientapi/sync.go b/server/internal/clientapi/sync.go index 8175f30..06ed8da 100644 --- a/server/internal/clientapi/sync.go +++ b/server/internal/clientapi/sync.go @@ -34,6 +34,8 @@ func syncError(err error) error { return httputil.InvalidParam("%s", err.Error()) case errors.Is(err, syncapi.ErrInvalidToken): return httputil.InvalidParam("%s", err.Error()) + case errors.Is(err, syncapi.ErrUnknownPos): + return httputil.NewError(http.StatusBadRequest, httputil.ErrCodeUnknownPos, "%s", err.Error()) case errors.Is(err, syncapi.ErrPresenceDisabled): return httputil.Forbidden("Presence is disabled on this server") diff --git a/server/internal/config/config.go b/server/internal/config/config.go index 40c54d5..ae9180f 100644 --- a/server/internal/config/config.go +++ b/server/internal/config/config.go @@ -11,6 +11,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/dariuszjaniak/warp/server/internal/storage" ) // Config is the whole server configuration. @@ -31,6 +33,7 @@ type Config struct { ServerNotices ServerNotices `yaml:"server_notices"` Logging Logging `yaml:"logging"` Metrics Metrics `yaml:"metrics"` + Tracing Tracing `yaml:"tracing"` // path is the file this config was read from, if any. path string @@ -108,6 +111,48 @@ type Database struct { // AutoMigrate runs pending migrations on startup. Turn it off to gate // schema changes behind an explicit `warp migrate`. AutoMigrate bool `yaml:"auto_migrate" env:"WARP_DB_AUTO_MIGRATE"` + + // PocketBase configures what the pocketbase driver adds over sqlite. + PocketBase PocketBase `yaml:"pocketbase"` +} + +// PocketBase configures the pocketbase storage driver. +type PocketBase struct { + // S3 puts media content in an S3-compatible bucket through + // PocketBase's filesystem, instead of under the data directory. + S3 PocketBaseS3 `yaml:"s3"` +} + +// PocketBaseS3 is an S3-compatible bucket. +type PocketBaseS3 struct { + Enabled bool `yaml:"enabled" env:"WARP_PB_S3_ENABLED"` + Bucket string `yaml:"bucket" env:"WARP_PB_S3_BUCKET"` + Region string `yaml:"region" env:"WARP_PB_S3_REGION"` + Endpoint string `yaml:"endpoint" env:"WARP_PB_S3_ENDPOINT"` + AccessKey string `yaml:"access_key" env:"WARP_PB_S3_ACCESS_KEY"` + Secret string `yaml:"secret" env:"WARP_PB_S3_SECRET"` + ForcePathStyle bool `yaml:"force_path_style" env:"WARP_PB_S3_FORCE_PATH_STYLE"` +} + +// Storage is the storage driver configuration this section describes. +func (d Database) Storage() storage.Config { + s3 := d.PocketBase.S3 + return storage.Config{ + Driver: d.Driver, + ConnectionString: d.ConnectionString, + MaxOpenConns: d.MaxOpenConns, + MaxIdleConns: d.MaxIdleConns, + ConnMaxLifetime: d.ConnMaxLifetime, + S3: storage.S3Config{ + Enabled: s3.Enabled, + Bucket: s3.Bucket, + Region: s3.Region, + Endpoint: s3.Endpoint, + AccessKey: s3.AccessKey, + Secret: s3.Secret, + ForcePathStyle: s3.ForcePathStyle, + }, + } } // ClientAPI configures the Client-Server API surface. @@ -391,6 +436,22 @@ type Metrics struct { Path string `yaml:"path" env:"WARP_METRICS_PATH"` } +// Tracing configures OpenTelemetry tracing. +type Tracing struct { + Enabled bool `yaml:"enabled" env:"WARP_TRACING_ENABLED"` + + // Endpoint is the OTLP/HTTP collector, e.g. "http://localhost:4318". + // Empty falls back to the standard OTEL_EXPORTER_OTLP_ENDPOINT. + Endpoint string `yaml:"endpoint" env:"WARP_TRACING_ENDPOINT"` + + // SampleRatio is the fraction of new traces kept, from 0 to 1. A + // request that arrives as part of a sampled trace is always kept. + SampleRatio float64 `yaml:"sample_ratio" env:"WARP_TRACING_SAMPLE_RATIO"` + + // ServiceName is what the traces are reported as. + ServiceName string `yaml:"service_name" env:"WARP_TRACING_SERVICE_NAME"` +} + // Path returns the file this config was loaded from, or "" for defaults. func (c *Config) Path() string { return c.path } @@ -481,6 +542,7 @@ func Default() *Config { }, Logging: Logging{Level: "info", Format: "text"}, Metrics: Metrics{Enabled: true, Path: "/metrics"}, + Tracing: Tracing{SampleRatio: 1, ServiceName: "warp"}, } } @@ -571,6 +633,20 @@ func (c *Config) Validate() error { if c.Metrics.Enabled && !strings.HasPrefix(c.Metrics.Path, "/") { bad("metrics.path: must start with /") } + if c.Tracing.SampleRatio < 0 || c.Tracing.SampleRatio > 1 { + bad("tracing.sample_ratio: must be between 0 and 1") + } + if c.Tracing.Endpoint != "" { + if u, err := url.Parse(c.Tracing.Endpoint); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + bad("tracing.endpoint: must be an http or https URL") + } + } + if c.Database.Driver == "pocketbase" && c.Database.PocketBase.S3.Enabled { + s3 := c.Database.PocketBase.S3 + if s3.Bucket == "" || s3.Region == "" || s3.Endpoint == "" || s3.AccessKey == "" || s3.Secret == "" { + bad("database.pocketbase.s3: bucket, region, endpoint, access_key and secret are all required") + } + } if c.ClientAPI.AccessTokenLifetime < 0 { bad("client_api.access_token_lifetime: must not be negative") diff --git a/server/internal/config/env.go b/server/internal/config/env.go index 0273ba7..9dc7f69 100644 --- a/server/internal/config/env.go +++ b/server/internal/config/env.go @@ -76,6 +76,12 @@ func setFromString(fv reflect.Value, raw string) error { return fmt.Errorf("invalid integer %q", raw) } fv.SetInt(n) + case reflect.Float32, reflect.Float64: + f, err := strconv.ParseFloat(raw, 64) + if err != nil { + return fmt.Errorf("invalid number %q", raw) + } + fv.SetFloat(f) case reflect.Slice: if fv.Type().Elem().Kind() != reflect.String { return fmt.Errorf("unsupported slice type %s", fv.Type()) diff --git a/server/internal/federation/client.go b/server/internal/federation/client.go index 08e995c..8e8dcd3 100644 --- a/server/internal/federation/client.go +++ b/server/internal/federation/client.go @@ -17,6 +17,7 @@ import ( "github.com/dariuszjaniak/warp/server/internal/buildinfo" "github.com/dariuszjaniak/warp/server/internal/matrix/signing" + "github.com/dariuszjaniak/warp/server/internal/tracing" ) // Errors an outbound request can fail with before anything is sent. @@ -170,7 +171,8 @@ func NewClient(opts ClientOptions) *Client { key: opts.Key, resolve: opts.Resolve, http: &http.Client{ - Transport: transport, + // Each request is a client span, and carries the trace on. + Transport: tracing.Transport(transport), // Federation endpoints do not redirect; following one would // send a signed request somewhere it was not addressed. CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, @@ -551,6 +553,20 @@ func (c *Client) PublicRooms(ctx context.Context, destination string, limit int, return out, err } +// Hierarchy asks a server about one room of a space and its children. +func (c *Client) Hierarchy(ctx context.Context, destination, roomID string, suggestedOnly bool) (json.RawMessage, error) { + query := url.Values{} + if suggestedOnly { + query.Set("suggested_only", "true") + } + var out json.RawMessage + err := c.do(ctx, request{ + method: http.MethodGet, destination: destination, signed: true, + path: "/_matrix/federation/v1/hierarchy/" + url.PathEscape(roomID), query: query, + }, &out) + return out, err +} + // Version asks a server what software it runs. func (c *Client) Version(ctx context.Context, destination string) (VersionResponse, error) { var out VersionResponse diff --git a/server/internal/federationapi/server.go b/server/internal/federationapi/server.go index b106626..8a561d0 100644 --- a/server/internal/federationapi/server.go +++ b/server/internal/federationapi/server.go @@ -157,6 +157,7 @@ func (s *Server) RegisterRoutes(r *httputil.Router) { authed("GET "+v1+"/query/profile", s.handleQueryProfile) authed("GET "+v1+"/publicRooms", s.handlePublicRoomsGet) authed("POST "+v1+"/publicRooms", s.handlePublicRoomsPost) + authed("GET "+v1+"/hierarchy/{roomID}", s.handleHierarchy) s.registerMediaRoutes(authed) s.registerKeyRoutes(authed) @@ -668,6 +669,19 @@ func (s *Server) publicRooms(w http.ResponseWriter, r *http.Request, req roomser return nil } +// handleHierarchy implements GET /_matrix/federation/v1/hierarchy/{roomID}: +// another server walking one of this server's spaces. +func (s *Server) handleHierarchy(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error { + roomID := r.PathValue("roomID") + suggestedOnly, _ := strconv.ParseBool(r.URL.Query().Get("suggested_only")) + resp, err := s.rooms.FederatedHierarchy(r.Context(), origin, roomID, suggestedOnly) + if err != nil { + return fedError(err) + } + httputil.WriteJSON(w, http.StatusOK, resp) + return nil +} + // String is for logs. func (s *Server) String() string { return fmt.Sprintf("federation API for %s", s.serverName) } diff --git a/server/internal/httputil/errors.go b/server/internal/httputil/errors.go index ff0345e..9e49e09 100644 --- a/server/internal/httputil/errors.go +++ b/server/internal/httputil/errors.go @@ -49,6 +49,10 @@ const ( ErrCodeConnectionFailed = "M_CONNECTION_FAILED" ErrCodeConnectionTimeout = "M_CONNECTION_TIMEOUT" ErrCodeBadStatus = "M_BAD_STATUS" + + // Sliding sync, added in Phase 10: a `pos` the server no longer + // remembers, which tells the client to start a new connection. + ErrCodeUnknownPos = "M_UNKNOWN_POS" ) // MatrixError is the standard Matrix error envelope. Every non-2xx response diff --git a/server/internal/pushserver/notifications.go b/server/internal/pushserver/notifications.go index 558f42c..f59e515 100644 --- a/server/internal/pushserver/notifications.go +++ b/server/internal/pushserver/notifications.go @@ -24,19 +24,17 @@ const ( // ReceiptSet is told about a read receipt a local user set, and clears // their notifications in the room up to the event it names. // -// Notifications are not tracked per thread (threads are Phase 10), so a -// receipt counts when it is for the room as a whole or for its main -// timeline, and a receipt inside a thread is left alone: clearing the -// room on it would mark the main timeline read when the user only looked -// at one thread. Thread notifications are cleared when the main receipt -// moves past them. +// Notifications are filed by thread, and a receipt reads the part of the +// room it is for: thread_id "main" the main timeline, a thread root's ID +// that thread, and no thread_id the whole room. Reading one thread leaves +// the main timeline and the other threads unread, and a client that does +// not know about threads sends unthreaded receipts and keeps clearing the +// room as a whole, as before threads. func (a *API) ReceiptSet(ctx context.Context, userID, roomID, receiptType, threadID, eventID string) { if receiptType != receiptRead && receiptType != receiptReadPrivate { return } - if threadID != "" && threadID != threadMain { - return - } + thread := receiptThread(threadID) pos, err := a.rooms.EventPosition(ctx, roomID, userID, eventID) if err != nil { // A receipt for an event this server does not have — or one the @@ -47,16 +45,28 @@ func (a *API) ReceiptSet(ctx context.Context, userID, roomID, receiptType, threa } return } - if err := a.markRead(ctx, userID, roomID, pos); err != nil { + if err := a.markRead(ctx, userID, roomID, thread, pos); err != nil { a.log.Warn("clearing notifications on a receipt failed", "user_id", userID, "room_id", roomID, "error", err) } } -// markRead clears a user's notifications in a room up to a position, and -// wakes their syncs if the counts moved. -func (a *API) markRead(ctx context.Context, userID, roomID string, upTo int64) error { - _, moved, err := a.store.MarkNotificationsRead(ctx, userID, roomID, upTo) +// receiptThread is the part of a room a receipt's thread_id reads, as the +// store names it. +func receiptThread(threadID string) string { + switch threadID { + case "": + return storage.AllThreads + case threadMain: + return storage.ThreadMain + } + return threadID +} + +// markRead clears a user's notifications in a room, or in one thread of +// it, up to a position, and wakes their syncs if the counts moved. +func (a *API) markRead(ctx context.Context, userID, roomID, thread string, upTo int64) error { + _, moved, err := a.store.MarkNotificationsRead(ctx, userID, roomID, thread, upTo) if err != nil { return err } @@ -71,6 +81,12 @@ func (a *API) NotificationCounts(ctx context.Context, userID string, roomIDs []s return a.store.NotificationCounts(ctx, userID, roomIDs) } +// ThreadNotificationCounts reads a user's unread counts in the given rooms +// per thread, with the main timeline under storage.ThreadMain. +func (a *API) ThreadNotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]map[string]storage.NotificationCounts, error) { + return a.store.ThreadNotificationCounts(ctx, userID, roomIDs) +} + // NotificationCountChanges lists the rooms whose counts for a user moved // in a window of the notification stream. func (a *API) NotificationCountChanges(ctx context.Context, userID string, from, to int64) ([]string, error) { diff --git a/server/internal/pushserver/notifications_test.go b/server/internal/pushserver/notifications_test.go new file mode 100644 index 0000000..f9b2aa7 --- /dev/null +++ b/server/internal/pushserver/notifications_test.go @@ -0,0 +1,22 @@ +package pushserver + +import ( + "testing" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +func TestReceiptThread(t *testing.T) { + for threadID, want := range map[string]string{ + // No thread_id: the receipt reads the room as a whole. + "": storage.AllThreads, + // "main" reads the main timeline, which the store files as ''. + "main": storage.ThreadMain, + // Anything else is a thread root, and reads that thread. + "$root:warp.example": "$root:warp.example", + } { + if got := receiptThread(threadID); got != want { + t.Errorf("receiptThread(%q) = %q, want %q", threadID, got, want) + } + } +} diff --git a/server/internal/pushserver/pushserver.go b/server/internal/pushserver/pushserver.go index 0b6277c..c351dd5 100644 --- a/server/internal/pushserver/pushserver.go +++ b/server/internal/pushserver/pushserver.go @@ -28,6 +28,7 @@ import ( "github.com/dariuszjaniak/warp/server/internal/pushrules" "github.com/dariuszjaniak/warp/server/internal/roomserver" "github.com/dariuszjaniak/warp/server/internal/storage" + "github.com/dariuszjaniak/warp/server/internal/tracing" ) // Errors callers have to tell apart. The rule-language sentinels are @@ -174,7 +175,7 @@ func New(opts Options) (*API, error) { opts.NotificationRetention = DefaultNotificationRetention } if opts.HTTPClient == nil { - opts.HTTPClient = &http.Client{Timeout: opts.GatewayTimeout} + opts.HTTPClient = &http.Client{Timeout: opts.GatewayTimeout, Transport: tracing.Transport(nil)} } ctx, cancel := context.WithCancel(context.Background()) diff --git a/server/internal/pushserver/worker.go b/server/internal/pushserver/worker.go index 5a95ae1..0dbdc6e 100644 --- a/server/internal/pushserver/worker.go +++ b/server/internal/pushserver/worker.go @@ -104,7 +104,7 @@ func (a *API) process(ctx context.Context, n roomserver.Notification) error { if target, ok := e.StateKey(); ok && a.isLocal(target) { switch membership, _ := e.Membership(); membership { case event.MembershipLeave, event.MembershipBan: - if err := a.markRead(ctx, target, n.RoomID, n.StreamOrdering); err != nil { + if err := a.markRead(ctx, target, n.RoomID, storage.AllThreads, n.StreamOrdering); err != nil { return err } } @@ -144,6 +144,7 @@ func (a *API) process(ctx context.Context, n roomserver.Notification) error { var ( notifications []storage.NewNotification outcomes = map[string]pushrules.Outcome{} + thread = roomserver.ThreadOf(e) ) now := a.now().UnixMilli() for _, userID := range candidates { @@ -179,6 +180,7 @@ func (a *API) process(ctx context.Context, n roomserver.Notification) error { RoomID: n.RoomID, EventID: e.EventID(), EventPos: n.StreamOrdering, + ThreadID: thread, TimestampMS: now, Highlight: outcome.Highlight, Actions: actions, diff --git a/server/internal/roomserver/federated_test.go b/server/internal/roomserver/federated_test.go index df39753..fe8ae6d 100644 --- a/server/internal/roomserver/federated_test.go +++ b/server/internal/roomserver/federated_test.go @@ -346,6 +346,18 @@ func (t transport) QueryDirectory(ctx context.Context, dest, alias string) (fede return federation.DirectoryResponse{RoomID: resolved.RoomID, Servers: resolved.Servers}, httpError(err) } +func (t transport) Hierarchy(ctx context.Context, dest, roomID string, suggestedOnly bool) (json.RawMessage, error) { + api, err := t.to(dest) + if err != nil { + return nil, err + } + resp, err := api.FederatedHierarchy(ctx, t.from.name, roomID, suggestedOnly) + if err != nil { + return nil, httpError(err) + } + return json.Marshal(resp) +} + func (t transport) PublicRooms(ctx context.Context, dest string, limit int, since, search string) (json.RawMessage, error) { api, err := t.to(dest) if err != nil { @@ -1052,3 +1064,69 @@ func (m *mesh) setDisabled(disabled bool) { defer m.mu.Unlock() m.disabled = disabled } + +func TestFederatedSpaceHierarchy(t *testing.T) { + m := newMesh(t, serverA, serverB) + a, b := m.servers[serverA], m.servers[serverB] + ctx := t.Context() + + space, err := a.api.CreateRoom(ctx, roomserver.CreateRoomRequest{ + Creator: userA, Preset: roomserver.PresetPublicChat, Name: "Space", + CreationContent: json.RawMessage(`{"type":"m.space"}`), + }) + if err != nil { + t.Fatalf("CreateRoom space: %v", err) + } + public := a.publicRoom(t, userA) + private, err := a.api.CreateRoom(ctx, roomserver.CreateRoomRequest{Creator: userA, Name: "Private"}) + if err != nil { + t.Fatalf("CreateRoom private: %v", err) + } + for _, child := range []string{public, private.RoomID} { + key := child + if _, err := a.api.SendEvent(ctx, roomserver.SendEventRequest{ + RoomID: space.RoomID, Sender: userA, Type: roomserver.TypeSpaceChild, StateKey: &key, + Content: json.RawMessage(`{"via":["` + serverA + `"]}`), + }); err != nil { + t.Fatalf("add child: %v", err) + } + } + a.takeOutbox() + + if _, err := b.api.JoinRoom(ctx, roomserver.JoinRequest{ + RoomID: space.RoomID, User: userB, ServerNames: []string{serverA}, + }); err != nil { + t.Fatalf("remote join: %v", err) + } + if failures := m.flush(); len(failures) > 0 { + t.Fatalf("delivery failures: %v", failures) + } + + // B holds the space itself and asks A about the rooms it is not in: + // the public one comes back, the private one is A's to keep. + page, err := b.api.Hierarchy(ctx, roomserver.HierarchyRequest{RoomID: space.RoomID, User: userB, MaxDepth: -1}) + if err != nil { + t.Fatalf("Hierarchy: %v", err) + } + var got []string + for _, room := range page.Rooms { + got = append(got, room.RoomID) + } + if len(got) != 2 || got[0] != space.RoomID || got[1] != public { + t.Fatalf("B's walk of the space = %v, want the space and the public room", got) + } + if page.Rooms[1].Name != "Federated" || page.Rooms[1].NumJoinedMembers != 1 { + t.Errorf("the remote child = %+v", page.Rooms[1]) + } + + answer, err := a.api.FederatedHierarchy(ctx, serverB, space.RoomID, false) + if err != nil { + t.Fatalf("FederatedHierarchy: %v", err) + } + if len(answer.InaccessibleChildren) != 1 || answer.InaccessibleChildren[0] != private.RoomID { + t.Errorf("inaccessible_children = %v, want the private room", answer.InaccessibleChildren) + } + if _, err := a.api.FederatedHierarchy(ctx, serverB, private.RoomID, false); !errors.Is(err, roomserver.ErrNoSuchRoom) { + t.Errorf("FederatedHierarchy of the private room = %v, want ErrNoSuchRoom", err) + } +} diff --git a/server/internal/roomserver/hierarchy.go b/server/internal/roomserver/hierarchy.go new file mode 100644 index 0000000..b21da73 --- /dev/null +++ b/server/internal/roomserver/hierarchy.go @@ -0,0 +1,616 @@ +package roomserver + +import ( + "cmp" + "context" + "encoding/json" + "fmt" + "slices" + "strings" + + "github.com/dariuszjaniak/warp/server/internal/matrix/auth" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/id" +) + +// Spaces: a space is a room whose create event says "type": "m.space", +// and whose m.space.child state events name the rooms in it. Nothing about +// that is enforced by the authorisation rules — a child event is a pointer, +// and the room it points at need not know — so what the server provides is +// the walk: /hierarchy reads a space's tree breadth first, summarising each +// room the user is allowed to know about, and asks the space's other +// servers about the rooms this one is not in. +// +// The same walk, one level deep and judged for a server rather than a user, +// answers the federation endpoint other servers walk Warp's spaces with. + +// The space event types. +const ( + TypeSpaceChild = "m.space.child" + TypeSpaceParent = "m.space.parent" + + // RoomTypeSpace is m.room.create's content.type for a space. + RoomTypeSpace = "m.space" +) + +// Limits of a hierarchy walk. +const ( + // DefaultHierarchyLimit is the page size when the client names none, + // and MaxHierarchyLimit caps the one it names. + DefaultHierarchyLimit = 50 + MaxHierarchyLimit = 100 + + // MaxHierarchyDepth caps max_depth: a space tree deeper than this is + // either a mistake or an attempt to make the server walk forever. + MaxHierarchyDepth = 50 + + // maxHierarchyRooms caps how many rooms one walk visits, whatever the + // page size: paging re-walks from the root, so the cap is what bounds + // the work of the last page of a large space. + maxHierarchyRooms = 2000 + + // maxOrderLength is the longest m.space.child order the spec lets + // sort a child; a longer one is ignored, as if absent. + maxOrderLength = 50 +) + +// SpaceChildEvent is an m.space.child event as a hierarchy carries it: the +// stripped state event plus its timestamp, which children are sorted by. +type SpaceChildEvent struct { + Type string `json:"type"` + StateKey string `json:"state_key"` + Sender string `json:"sender"` + Content json.RawMessage `json:"content"` + OriginServerTS int64 `json:"origin_server_ts"` +} + +// SpaceRoom is one room of a hierarchy: the public face of the room, as +// the public room directory shows it, and the space's children when it has +// any. +type SpaceRoom struct { + RoomID string `json:"room_id"` + Name string `json:"name,omitempty"` + Topic string `json:"topic,omitempty"` + AvatarURL string `json:"avatar_url,omitempty"` + CanonicalAlias string `json:"canonical_alias,omitempty"` + NumJoinedMembers int `json:"num_joined_members"` + WorldReadable bool `json:"world_readable"` + GuestCanJoin bool `json:"guest_can_join"` + JoinRule string `json:"join_rule,omitempty"` + RoomType string `json:"room_type,omitempty"` + RoomVersion string `json:"room_version,omitempty"` + Encryption string `json:"encryption,omitempty"` + + // ChildrenState is the room's m.space.child events. It is always there + // on the rooms a client is given, empty for a room that is not a + // space, and left out of the children a federation response lists. + ChildrenState []SpaceChildEvent `json:"children_state,omitempty"` + + // AllowedRoomIDs is the rooms whose members may join under a + // restricted join rule. Only a federation response carries it: the + // asking server needs it to decide what its own users may see. + AllowedRoomIDs []string `json:"allowed_room_ids,omitempty"` +} + +// HierarchyRequest is one page of a space's tree. +type HierarchyRequest struct { + RoomID string + + // User is who is walking. Each room is shown only if they may know + // about it. + User string + + SuggestedOnly bool + + // MaxDepth is how far below the root to go; zero is the root alone. + // Negative means the server's own limit. + MaxDepth int + + // From is how many rooms of the walk earlier pages have already + // returned. + From int + + Limit int +} + +// HierarchyPage is one page of a space's tree. +type HierarchyPage struct { + Rooms []SpaceRoom + + // Next is the From of the next page, zero at the end. + Next int +} + +// FederationHierarchy is what GET /_matrix/federation/v1/hierarchy +// answers: the room, the children this server can describe, and the ones +// the asking server may not see. +type FederationHierarchy struct { + Room SpaceRoom `json:"room"` + Children []SpaceRoom `json:"children"` + InaccessibleChildren []string `json:"inaccessible_children"` +} + +// spaceChild is one parsed m.space.child of a room. +type spaceChild struct { + roomID string + via []string + order string + ts int64 + suggested bool +} + +// localSpaceRoom is a room this server is in, read for a hierarchy. +type localSpaceRoom struct { + room SpaceRoom + children []spaceChild + state auth.State +} + +// Hierarchy reads one page of a space's tree. +func (r *API) Hierarchy(ctx context.Context, req HierarchyRequest) (HierarchyPage, error) { + maxDepth := req.MaxDepth + if maxDepth < 0 || maxDepth > MaxHierarchyDepth { + maxDepth = MaxHierarchyDepth + } + limit := req.Limit + if limit <= 0 { + limit = DefaultHierarchyLimit + } + limit = min(limit, MaxHierarchyLimit) + + type queued struct { + roomID string + depth int + via []string + } + queue := []queued{{roomID: req.RoomID}} + visited := map[string]bool{} + remote := map[string]*FederationHierarchy{} + + var ( + out HierarchyPage + emitted int + ) + for len(queue) > 0 && len(visited) < maxHierarchyRooms { + next := queue[0] + queue = queue[1:] + if visited[next.roomID] { + continue + } + visited[next.roomID] = true + isRoot := next.roomID == req.RoomID + + room, children, accessible, err := r.hierarchyRoom(ctx, req.User, next.roomID, next.via, req.SuggestedOnly, remote) + switch { + case err != nil && isRoot: + return HierarchyPage{}, err + case err != nil: + // A child this server cannot describe — not here, and no + // server it names would say — is left out, as the spec has + // it, rather than failing the walk. + r.log.Debug("left a room out of a space hierarchy", + "space", req.RoomID, "room_id", next.roomID, "error", err) + continue + case !accessible && isRoot: + return HierarchyPage{}, fmt.Errorf("%w: %s may not see the space %s", ErrForbidden, req.User, req.RoomID) + case !accessible: + continue + } + + if emitted >= req.From { + if len(out.Rooms) == limit { + out.Next = emitted + return out, nil + } + out.Rooms = append(out.Rooms, room) + } + emitted++ + + // Only a space's children are walked into: an ordinary room with + // an m.space.child event is not a space, and the event in it is + // an orphan. + if next.depth >= maxDepth || (!isRoot && room.RoomType != RoomTypeSpace) { + continue + } + for _, child := range children { + if !visited[child.roomID] { + queue = append(queue, queued{roomID: child.roomID, depth: next.depth + 1, via: child.via}) + } + } + } + return out, nil +} + +// hierarchyRoom describes one room of a walk, and reports whether the user +// may know about it. +func (r *API) hierarchyRoom(ctx context.Context, userID, roomID string, via []string, suggestedOnly bool, remote map[string]*FederationHierarchy) (SpaceRoom, []spaceChild, bool, error) { + local, err := r.participating(ctx, roomID) + if err != nil { + return SpaceRoom{}, nil, false, err + } + if local { + read, err := r.readSpaceRoom(ctx, roomID, suggestedOnly) + if err != nil { + return SpaceRoom{}, nil, false, err + } + accessible, err := r.userMayPeek(ctx, read, userID) + if err != nil { + return SpaceRoom{}, nil, false, err + } + read.room.AllowedRoomIDs = nil + return read.room, read.children, accessible, nil + } + + answer, err := r.remoteHierarchy(ctx, roomID, via, suggestedOnly, remote) + if err != nil { + return SpaceRoom{}, nil, false, err + } + room := answer.Room + children := parseRemoteChildren(room.ChildrenState, suggestedOnly) + if room.ChildrenState == nil { + room.ChildrenState = []SpaceChildEvent{} + } + + // The other server judged the room for this server as a whole; a + // restricted room is shown to the user only if they are in one of the + // rooms it lets in, which only this server knows. + accessible := true + if isRestricted(room.JoinRule) && !isPeekable(room.JoinRule, room.WorldReadable) { + accessible = false + for _, allowed := range room.AllowedRoomIDs { + membership, err := r.store.Membership(ctx, allowed, userID) + if err != nil { + return SpaceRoom{}, nil, false, err + } + if membership.Membership == event.MembershipJoin { + accessible = true + break + } + } + } + room.AllowedRoomIDs = nil + return room, children, accessible, nil +} + +// remoteHierarchy asks the servers a child event names about a room, +// remembering the answer for the rest of the walk. +func (r *API) remoteHierarchy(ctx context.Context, roomID string, via []string, suggestedOnly bool, cache map[string]*FederationHierarchy) (*FederationHierarchy, error) { + if cached, ok := cache[roomID]; ok { + if cached == nil { + return nil, fmt.Errorf("%w: %s", ErrNoSuchRoom, roomID) + } + return cached, nil + } + cache[roomID] = nil + if !r.federating() || r.remote == nil { + return nil, fmt.Errorf("%w: %s is not a room this server is in", ErrNoSuchRoom, roomID) + } + + var lastErr error + for _, server := range via { + if server == r.serverName { + continue + } + raw, err := r.remote.Hierarchy(ctx, server, roomID, suggestedOnly) + if err != nil { + lastErr = remoteError(server, err) + continue + } + var answer FederationHierarchy + if err := json.Unmarshal(raw, &answer); err != nil || answer.Room.RoomID != roomID { + lastErr = fmt.Errorf("%w: %s sent an unreadable hierarchy for %s", ErrRemote, server, roomID) + continue + } + cache[roomID] = &answer + return &answer, nil + } + if lastErr == nil { + lastErr = fmt.Errorf("%w: no server to ask about %s", ErrNoSuchRoom, roomID) + } + return nil, lastErr +} + +// FederatedHierarchy answers another server walking a space of this one. +func (r *API) FederatedHierarchy(ctx context.Context, origin, roomID string, suggestedOnly bool) (FederationHierarchy, error) { + local, err := r.participating(ctx, roomID) + if err != nil { + return FederationHierarchy{}, err + } + if !local { + return FederationHierarchy{}, fmt.Errorf("%w: %s", ErrNoSuchRoom, roomID) + } + root, err := r.readSpaceRoom(ctx, roomID, suggestedOnly) + if err != nil { + return FederationHierarchy{}, err + } + accessible, err := r.serverMayPeek(ctx, root, origin) + if err != nil { + return FederationHierarchy{}, err + } + if !accessible { + // Not found rather than forbidden: whether this server is in a + // room the asker may not see is itself not theirs to know. + return FederationHierarchy{}, fmt.Errorf("%w: %s", ErrNoSuchRoom, roomID) + } + + out := FederationHierarchy{Room: root.room, Children: []SpaceRoom{}, InaccessibleChildren: []string{}} + for _, child := range root.children { + local, err := r.participating(ctx, child.roomID) + if err != nil { + return FederationHierarchy{}, err + } + if !local { + continue + } + read, err := r.readSpaceRoom(ctx, child.roomID, suggestedOnly) + if err != nil { + return FederationHierarchy{}, err + } + ok, err := r.serverMayPeek(ctx, read, origin) + if err != nil { + return FederationHierarchy{}, err + } + if !ok { + out.InaccessibleChildren = append(out.InaccessibleChildren, child.roomID) + continue + } + read.room.ChildrenState = nil + out.Children = append(out.Children, read.room) + } + return out, nil +} + +// RoomSummaryRequest asks for the public face of one room. +type RoomSummaryRequest struct { + RoomID string + + // User is who is asking, empty for a guest without a token. + User string + + // Via are servers to ask when this one is not in the room. + Via []string +} + +// RoomSummaryResponse is GET /room_summary's answer: a hierarchy room +// without its children, plus where the user stands. +type RoomSummaryResponse struct { + SpaceRoom + Membership string `json:"membership,omitempty"` +} + +// RoomSummary describes one room to a user who may not be in it, as a link +// preview or an invite needs it. +func (r *API) RoomSummary(ctx context.Context, req RoomSummaryRequest) (RoomSummaryResponse, error) { + room, _, accessible, err := r.hierarchyRoom(ctx, req.User, req.RoomID, req.Via, false, map[string]*FederationHierarchy{}) + if err != nil { + return RoomSummaryResponse{}, err + } + if !accessible { + return RoomSummaryResponse{}, fmt.Errorf("%w: %s may not see %s", ErrForbidden, req.User, req.RoomID) + } + room.ChildrenState = nil + out := RoomSummaryResponse{SpaceRoom: room} + if req.User != "" { + membership, err := r.store.Membership(ctx, req.RoomID, req.User) + if err != nil && !isNotFound(err) { + return RoomSummaryResponse{}, err + } + out.Membership = membership.Membership + if out.Membership == "" { + out.Membership = event.MembershipLeave + } + } + return out, nil +} + +// readSpaceRoom reads a room this server is in. +func (r *API) readSpaceRoom(ctx context.Context, roomID string, suggestedOnly bool) (localSpaceRoom, error) { + room, rules, err := r.roomAndRules(ctx, roomID) + if err != nil { + return localSpaceRoom{}, err + } + summary, err := r.store.RoomSummary(ctx, roomID) + if err != nil { + return localSpaceRoom{}, err + } + st, err := r.currentState(ctx, room, rules) + if err != nil { + return localSpaceRoom{}, err + } + + out := localSpaceRoom{ + state: st, + room: SpaceRoom{ + RoomID: roomID, + Name: summary.Name, + Topic: summary.Topic, + AvatarURL: summary.AvatarURL, + CanonicalAlias: summary.CanonicalAlias, + NumJoinedMembers: summary.JoinedMembers, + WorldReadable: summary.HistoryVisibility == HistoryVisibilityWorldReadable, + GuestCanJoin: summary.GuestAccess == "can_join", + JoinRule: cmp.Or(st.JoinRule(), auth.JoinRuleInvite), + RoomType: summary.RoomType, + RoomVersion: room.Version, + ChildrenState: []SpaceChildEvent{}, + }, + } + if enc := st.Get(TypeEncryption, ""); enc != nil { + var content struct { + Algorithm string `json:"algorithm"` + } + if json.Unmarshal(enc.Content(), &content) == nil { + out.room.Encryption = content.Algorithm + } + } + if isRestricted(out.room.JoinRule) { + if out.room.AllowedRoomIDs, err = allowedRooms(st); err != nil { + return localSpaceRoom{}, err + } + } + + for key, e := range st { + if key.Type != TypeSpaceChild { + continue + } + child, ok := parseSpaceChild(key.StateKey, e.Content(), e.OriginServerTS()) + if !ok || (suggestedOnly && !child.suggested) { + continue + } + out.children = append(out.children, child) + out.room.ChildrenState = append(out.room.ChildrenState, SpaceChildEvent{ + Type: TypeSpaceChild, + StateKey: key.StateKey, + Sender: e.Sender(), + Content: e.Content(), + OriginServerTS: e.OriginServerTS(), + }) + } + sortChildren(out.children) + slices.SortFunc(out.room.ChildrenState, func(a, b SpaceChildEvent) int { + return strings.Compare(a.StateKey, b.StateKey) + }) + return out, nil +} + +// userMayPeek reports whether a user may know about a room they are not +// necessarily in: the spec's accessibility rule for a hierarchy. +func (r *API) userMayPeek(ctx context.Context, read localSpaceRoom, userID string) (bool, error) { + switch read.state.Membership(userID) { + case event.MembershipJoin, event.MembershipInvite: + return true, nil + } + if isPeekable(read.room.JoinRule, read.room.WorldReadable) { + return true, nil + } + if isRestricted(read.room.JoinRule) { + for _, allowed := range read.room.AllowedRoomIDs { + membership, err := r.store.Membership(ctx, allowed, userID) + if err != nil { + return false, err + } + if membership.Membership == event.MembershipJoin { + return true, nil + } + } + } + return false, nil +} + +// serverMayPeek is userMayPeek for another server: anything a user of it +// could be shown. A restricted room is shown with its allow list, which +// the asking server checks against its own users. +func (r *API) serverMayPeek(ctx context.Context, read localSpaceRoom, origin string) (bool, error) { + if isPeekable(read.room.JoinRule, read.room.WorldReadable) || isRestricted(read.room.JoinRule) { + return true, nil + } + for key, e := range read.state { + if key.Type != event.TypeMember { + continue + } + server, err := id.ServerNameOf(key.StateKey) + if err != nil || server != origin { + continue + } + if membership, err := e.Membership(); err == nil && + (membership == event.MembershipJoin || membership == event.MembershipInvite) { + return true, nil + } + } + return false, nil +} + +// isPeekable reports whether anybody at all may know about a room. +func isPeekable(joinRule string, worldReadable bool) bool { + switch joinRule { + case auth.JoinRulePublic, auth.JoinRuleKnock, auth.JoinRuleKnockRestricted: + return true + } + return worldReadable +} + +func isRestricted(joinRule string) bool { + return joinRule == auth.JoinRuleRestricted || joinRule == auth.JoinRuleKnockRestricted +} + +// parseSpaceChild reads one m.space.child. A child event without a +// non-empty `via` is how a space removes a child — the event cannot be +// deleted, only emptied — so it does not count. +func parseSpaceChild(stateKey string, content json.RawMessage, ts int64) (spaceChild, bool) { + if !strings.HasPrefix(stateKey, "!") { + return spaceChild{}, false + } + var c struct { + Via []any `json:"via"` + Order any `json:"order"` + Suggested any `json:"suggested"` + } + if err := json.Unmarshal(content, &c); err != nil { + return spaceChild{}, false + } + child := spaceChild{roomID: stateKey, ts: ts} + for _, v := range c.Via { + if s, ok := v.(string); ok && s != "" { + child.via = append(child.via, s) + } + } + if len(child.via) == 0 { + return spaceChild{}, false + } + if order, ok := c.Order.(string); ok && validOrder(order) { + child.order = order + } + child.suggested, _ = c.Suggested.(bool) + return child, true +} + +// parseRemoteChildren reads the children of a room another server +// described. +func parseRemoteChildren(state []SpaceChildEvent, suggestedOnly bool) []spaceChild { + var out []spaceChild + for _, e := range state { + if e.Type != TypeSpaceChild { + continue + } + child, ok := parseSpaceChild(e.StateKey, e.Content, e.OriginServerTS) + if !ok || (suggestedOnly && !child.suggested) { + continue + } + out = append(out, child) + } + sortChildren(out) + return out +} + +// validOrder reports whether an order string may sort a child: printable +// ASCII and no longer than the spec allows. +func validOrder(order string) bool { + if order == "" || len(order) > maxOrderLength { + return false + } + for i := range len(order) { + if order[i] < 0x20 || order[i] > 0x7e { + return false + } + } + return true +} + +// sortChildren puts children in the spec's order: those with an order +// first, by it; then by when the child event was sent; then by room ID, +// which only breaks ties. +func sortChildren(children []spaceChild) { + slices.SortStableFunc(children, func(a, b spaceChild) int { + switch { + case a.order != "" && b.order == "": + return -1 + case a.order == "" && b.order != "": + return 1 + } + return cmp.Or( + strings.Compare(a.order, b.order), + cmp.Compare(a.ts, b.ts), + strings.Compare(a.roomID, b.roomID), + ) + }) +} diff --git a/server/internal/roomserver/input.go b/server/internal/roomserver/input.go index 4d0956a..36d575d 100644 --- a/server/internal/roomserver/input.go +++ b/server/internal/roomserver/input.go @@ -6,11 +6,14 @@ import ( "errors" "fmt" + "go.opentelemetry.io/otel/attribute" + "github.com/dariuszjaniak/warp/server/internal/matrix/auth" "github.com/dariuszjaniak/warp/server/internal/matrix/event" "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" "github.com/dariuszjaniak/warp/server/internal/matrix/stateres" "github.com/dariuszjaniak/warp/server/internal/storage" + "github.com/dariuszjaniak/warp/server/internal/tracing" ) // sendRequest is one event this server is about to create. @@ -344,20 +347,17 @@ type persistRequest struct { } // persist writes the event and everything its arrival changes, then emits. -func (r *API) persist(ctx context.Context, req persistRequest) error { +func (r *API) persist(ctx context.Context, req persistRequest) (err error) { + ctx, span := tracing.Start(ctx, "roomserver.persist", + attribute.String("matrix.room_id", req.Room.ID), + attribute.String("matrix.event_type", req.Event.Type())) + defer func() { tracing.End(span, err) }() + e := req.Event stateKey, isState := e.StateKey() write := storage.RoomWrite{ - Event: storage.StoredEvent{ - RoomID: req.Room.ID, - EventID: e.EventID(), - Type: e.Type(), - Sender: e.Sender(), - Depth: e.Depth(), - OriginServerTS: e.OriginServerTS(), - JSON: e.JSON(), - }, + Event: storedOf(req.Room.ID, e), StateBefore: stateEntries(req.StateBefore), Extremities: req.Extremities, Transaction: req.Transaction, @@ -365,9 +365,6 @@ func (r *API) persist(ctx context.Context, req persistRequest) error { if write.Extremities == nil { write.Extremities = []string{e.EventID()} } - if isState { - write.Event.StateKey = &stateKey - } // The state after this event. For the ordinary case that is one tuple // to upsert; after a resolution it is the whole state, which the store diff --git a/server/internal/roomserver/query.go b/server/internal/roomserver/query.go index 54f60bd..bc2f178 100644 --- a/server/internal/roomserver/query.go +++ b/server/internal/roomserver/query.go @@ -129,7 +129,11 @@ func (r *API) Event(ctx context.Context, roomID, userID, eventID string) (Client if err != nil { return ClientEvent{}, err } - return r.clientEvent(stored, e, roomID), nil + out := []ClientEvent{r.clientEvent(stored, e, roomID)} + if err := r.bundleAggregations(ctx, roomID, rules, userID, visible, []storage.StoredEvent{stored}, out); err != nil { + return ClientEvent{}, err + } + return out[0], nil } // RoomState returns a room's state in the client shape: the current state diff --git a/server/internal/roomserver/relations.go b/server/internal/roomserver/relations.go new file mode 100644 index 0000000..e186a39 --- /dev/null +++ b/server/internal/roomserver/relations.go @@ -0,0 +1,475 @@ +package roomserver + +import ( + "context" + "encoding/json" + "fmt" + "slices" + + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/matrix/roomversion" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// Relations: threads, edits, reactions and references. +// +// An event relates to another through content.m.relates_to, and nothing +// about that is checked by the authorisation rules — a relation is a +// client convention the server indexes, not part of the room's DAG. The +// server's part is three things: recording which event points at which +// (relationOf, written with the event), listing an event's children +// (/relations and /threads), and summarising them into the parent's +// unsigned.m.relations wherever the parent is served (the bundled +// aggregations), so that a client rendering a timeline knows a message +// has a thread under it or has been edited without asking about every +// event. + +// relationOf reads content.m.relates_to. Both fields have to be strings, +// and an m.in_reply_to alone is not a relation in this sense: a reply +// points at an event without being one of its children. +func relationOf(content json.RawMessage) (eventID, relType string) { + var probe struct { + RelatesTo *struct { + EventID any `json:"event_id"` + RelType any `json:"rel_type"` + } `json:"m.relates_to"` + } + if err := json.Unmarshal(content, &probe); err != nil || probe.RelatesTo == nil { + return "", "" + } + id, ok1 := probe.RelatesTo.EventID.(string) + rel, ok2 := probe.RelatesTo.RelType.(string) + if !ok1 || !ok2 || id == "" || rel == "" { + return "", "" + } + return id, rel +} + +// ThreadOf is the thread an event is in: the root its m.relates_to names +// when the relation is m.thread, or "" for an event on the main timeline. +// The push server files each notification under it. +func ThreadOf(e *event.Event) string { + if root, rel := relationOf(e.Content()); rel == storage.RelThread { + return root + } + return "" +} + +// searchableText is what the search index holds for an event: the body of +// a message, the name of a room or its topic, as the spec's three search +// keys. An encrypted message has no body the server can read and is never +// found, which is what end-to-end encryption means. +func searchableText(e *event.Event) (key, text string) { + var field, k string + switch e.Type() { + case TypeMessage: + field, k = "body", "content.body" + case TypeName: + field, k = "name", "content.name" + case TypeTopic: + field, k = "topic", "content.topic" + default: + return "", "" + } + var content map[string]any + if err := json.Unmarshal(e.Content(), &content); err != nil { + return "", "" + } + value, ok := content[field].(string) + if !ok || value == "" { + return "", "" + } + return k, value +} + +// maxRecursionDepth is how far a recursive /relations walks: the spec's +// own suggestion, which covers a reaction to an edit in a thread. +const maxRecursionDepth = 3 + +// maxReferenceChunk caps the event IDs in an m.reference summary. +const maxReferenceChunk = 50 + +// RelationsRequest is one page of an event's children. +type RelationsRequest struct { + RoomID string + EventID string + + // User is who is asking. The parent has to be visible to them, and + // so does every child. + User string + + // RelType and EventType narrow the children, when set. + RelType string + EventType string + + // Low, High and Direction are a TimelineRequest's. + Low int64 + High int64 + Direction storage.TimelineDirection + + Limit int + + // Recurse includes the children's children, down to + // maxRecursionDepth levels. + Recurse bool +} + +// RelationsPage is one page of an event's children. +type RelationsPage struct { + Timeline + + // Depth is how many levels the read went down: 1 without recursion. + Depth int +} + +// Relations reads one page of an event's children. +func (r *API) Relations(ctx context.Context, req RelationsRequest) (RelationsPage, error) { + _, rules, err := r.roomAndRules(ctx, req.RoomID) + if err != nil { + return RelationsPage{}, err + } + visible, err := r.requireReadable(ctx, req.RoomID, req.User) + if err != nil { + return RelationsPage{}, err + } + parent, err := r.store.Event(ctx, req.RoomID, req.EventID) + if err != nil { + return RelationsPage{}, err + } + if parent.Outlier || !visible.Contains(parent.StreamOrdering) { + return RelationsPage{}, fmt.Errorf("event %q: %w", req.EventID, ErrNoSuchEvent) + } + + // The parents to read the children of: the event, and with recursion + // every descendant down to the depth limit. The filters apply at every + // level, as the spec has it, so a walk for m.thread children only + // follows threads. + // + // The children of one level are the next level, so the walk adds a + // level to the parents only while it is above the limit: the events at + // the limit are read as children and are nobody's parents. depth is + // the deepest level that had anything in it. + parents := []string{req.EventID} + depth := 1 + if req.Recurse { + depth = 0 + level := parents + seen := map[string]bool{req.EventID: true} + for depth < maxRecursionDepth { + children, err := r.store.ChildEventIDs(ctx, req.RoomID, level, req.RelType, req.EventType) + if err != nil { + return RelationsPage{}, err + } + // A relation loop cannot be built — an event ID is a hash of + // content naming its parent — but a repeat across levels can. + children = slices.DeleteFunc(children, func(id string) bool { + if seen[id] { + return true + } + seen[id] = true + return false + }) + if len(children) == 0 { + break + } + depth++ + if depth < maxRecursionDepth { + parents = append(parents, children...) + } + level = children + } + depth = max(depth, 1) + } + + empty := Timeline{Start: req.Low, End: req.Low} + if req.Direction == storage.DirectionBackward && req.High != 0 { + empty = Timeline{Start: req.High, End: req.High} + } + low, high, any := visible.Clamp(req.Low, req.High) + if !any || req.Limit <= 0 { + return RelationsPage{Timeline: empty, Depth: depth}, nil + } + + stored, err := r.store.Relations(ctx, storage.RelationsRequest{ + RoomID: req.RoomID, + ParentIDs: parents, + RelType: req.RelType, + EventType: req.EventType, + Low: low, + High: high, + Direction: req.Direction, + Limit: req.Limit + 1, + }) + if err != nil { + return RelationsPage{}, err + } + + out := RelationsPage{Timeline: empty, Depth: depth} + if len(stored) > req.Limit { + out.Limited = true + stored = stored[:req.Limit] + } + if len(stored) == 0 { + return out, nil + } + if req.Direction == storage.DirectionBackward { + slices.Reverse(stored) + } + if out.Events, err = r.clientEvents(stored, rules, req.RoomID); err != nil { + return RelationsPage{}, err + } + if err := r.bundleAggregations(ctx, req.RoomID, rules, req.User, visible, stored, out.Events); err != nil { + return RelationsPage{}, err + } + out.Start = storage.PositionBefore(stored[0].StreamOrdering) + out.End = stored[len(stored)-1].StreamOrdering + return out, nil +} + +// ThreadsRequest is one page of a room's threads. +type ThreadsRequest struct { + RoomID string + User string + + // Participated keeps only the threads the user started or replied in. + Participated bool + + // Before is where the previous page ended, zero for the first. + Before int64 + + Limit int +} + +// ThreadsPage is one page of a room's threads: their root events, newest +// activity first, each with its m.thread summary. +type ThreadsPage struct { + Roots []ClientEvent + + // Next is the Before of the next page, zero at the end. + Next int64 +} + +// Threads reads one page of a room's threads. +func (r *API) Threads(ctx context.Context, req ThreadsRequest) (ThreadsPage, error) { + _, rules, err := r.roomAndRules(ctx, req.RoomID) + if err != nil { + return ThreadsPage{}, err + } + visible, err := r.requireReadable(ctx, req.RoomID, req.User) + if err != nil { + return ThreadsPage{}, err + } + if req.Limit <= 0 { + return ThreadsPage{}, nil + } + + participant := "" + if req.Participated { + participant = req.User + } + threads, err := r.store.Threads(ctx, storage.ThreadsRequest{ + RoomID: req.RoomID, + ParticipatedBy: participant, + Before: req.Before, + High: visible.To, + Limit: req.Limit + 1, + }) + if err != nil { + return ThreadsPage{}, err + } + + var out ThreadsPage + if len(threads) > req.Limit { + threads = threads[:req.Limit] + out.Next = threads[len(threads)-1].LatestOrdering + } + + ids := make([]string, 0, len(threads)) + for _, t := range threads { + ids = append(ids, t.RootID) + } + stored, err := r.store.EventsByID(ctx, req.RoomID, ids) + if err != nil { + return ThreadsPage{}, err + } + byID := make(map[string]storage.StoredEvent, len(stored)) + for _, s := range stored { + byID[s.EventID] = s + } + + // In the order the store ranked them, leaving out any root the user may + // not see: a thread started before a `joined` user's join is not + // theirs to read, even if it is still going. + roots := make([]storage.StoredEvent, 0, len(threads)) + for _, t := range threads { + s, ok := byID[t.RootID] + if !ok || s.Outlier || !visible.Contains(s.StreamOrdering) { + continue + } + roots = append(roots, s) + } + if out.Roots, err = r.clientEvents(roots, rules, req.RoomID); err != nil { + return ThreadsPage{}, err + } + if err := r.bundleAggregations(ctx, req.RoomID, rules, req.User, visible, roots, out.Roots); err != nil { + return ThreadsPage{}, err + } + return out, nil +} + +// threadSummary is unsigned.m.relations.m.thread. +type threadSummary struct { + LatestEvent ClientEvent `json:"latest_event"` + Count int `json:"count"` + CurrentUserParticipated bool `json:"current_user_participated"` +} + +// referenceSummary is unsigned.m.relations.m.reference. +type referenceSummary struct { + Chunk []referenceEntry `json:"chunk"` +} + +type referenceEntry struct { + EventID string `json:"event_id"` +} + +// bundleAggregations adds unsigned.m.relations to the events of a read: +// the thread under each of them, its latest edit, and what references it. +// +// stored and events are the same events, in the same order — the rows and +// their client shape — because the summaries need what only the rows have +// (whether an event was redacted) and are written into the client shape. +// Annotations are not summarised: the spec dropped their aggregation, and a +// client counts reactions from the events themselves. +func (r *API) bundleAggregations(ctx context.Context, roomID string, rules roomversion.Rules, userID string, visible VisibleRange, stored []storage.StoredEvent, events []ClientEvent) error { + if len(stored) == 0 { + return nil + } + ids := make([]string, 0, len(stored)) + parents := make(map[string]storage.StoredEvent, len(stored)) + for _, s := range stored { + ids = append(ids, s.EventID) + parents[s.EventID] = s + } + + aggregates, err := r.store.RelationAggregates(ctx, roomID, ids, visible.To) + if err != nil { + return err + } + if len(aggregates) == 0 { + return nil + } + + type thread struct { + count int + latest int64 + latestID string + participated bool + } + threads := map[string]*thread{} + edits := map[string]storage.RelationAggregate{} + referenced := map[string]bool{} + for _, a := range aggregates { + parent := parents[a.ParentID] + switch a.RelType { + case storage.RelThread: + t := threads[a.ParentID] + if t == nil { + t = &thread{participated: parent.Sender == userID} + threads[a.ParentID] = t + } + t.count += a.Count + if a.LatestOrdering > t.latest { + t.latest, t.latestID = a.LatestOrdering, a.LatestEventID + } + if a.Sender == userID { + t.participated = true + } + case storage.RelReplace: + // Only the original's sender may edit it, a state event cannot + // be edited, and a redacted event's edits are moot: what they + // replaced is gone. + if a.Sender != parent.Sender || parent.IsState() || parent.RedactedBy != "" { + continue + } + edits[a.ParentID] = a + case storage.RelReference: + referenced[a.ParentID] = true + } + } + + // The latest thread replies and edits, read in one go. + latestIDs := make([]string, 0, len(threads)+len(edits)) + for _, t := range threads { + latestIDs = append(latestIDs, t.latestID) + } + for _, a := range edits { + latestIDs = append(latestIDs, a.LatestEventID) + } + latest, err := r.clientEventsByID(ctx, roomID, rules, latestIDs) + if err != nil { + return err + } + latestByID := make(map[string]ClientEvent, len(latest)) + for _, e := range latest { + latestByID[e.EventID] = e + } + + var references map[string][]string + if len(referenced) > 0 { + refParents := make([]string, 0, len(referenced)) + for id := range referenced { + refParents = append(refParents, id) + } + if references, err = r.store.RelationEventIDs(ctx, roomID, refParents, + storage.RelReference, visible.To, maxReferenceChunk); err != nil { + return err + } + } + + for i := range events { + id := events[i].EventID + relations := map[string]any{} + if t, ok := threads[id]; ok { + if e, ok := latestByID[t.latestID]; ok { + relations[storage.RelThread] = threadSummary{ + LatestEvent: e, + Count: t.count, + CurrentUserParticipated: t.participated, + } + } + } + if a, ok := edits[id]; ok { + // An edit has to keep the type of what it edits; one that does + // not is not a replacement the client should render. + if e, ok := latestByID[a.LatestEventID]; ok && e.Type == events[i].Type { + relations[storage.RelReplace] = e + } + } + if refs := references[id]; len(refs) > 0 { + summary := referenceSummary{Chunk: make([]referenceEntry, 0, len(refs))} + for _, ref := range refs { + summary.Chunk = append(summary.Chunk, referenceEntry{EventID: ref}) + } + relations[storage.RelReference] = summary + } + if len(relations) == 0 { + continue + } + events[i].Unsigned = r.withUnsigned(events[i].Unsigned, "m.relations", relations) + } + return nil +} + +// withUnsigned sets one key of an event's unsigned block. +func (r *API) withUnsigned(unsigned json.RawMessage, key string, value any) json.RawMessage { + block := map[string]json.RawMessage{} + if len(unsigned) > 0 { + if err := json.Unmarshal(unsigned, &block); err != nil { + r.log.Warn("an unsigned block this server built is unreadable", "error", err) + block = map[string]json.RawMessage{} + } + } + block[key] = mustMarshal(value) + return mustMarshal(block) +} diff --git a/server/internal/roomserver/remote.go b/server/internal/roomserver/remote.go index 93d737f..28545c1 100644 --- a/server/internal/roomserver/remote.go +++ b/server/internal/roomserver/remote.go @@ -37,6 +37,7 @@ type RemoteServers interface { Backfill(ctx context.Context, destination, roomID string, from []string, limit int) (federation.Transaction, error) QueryDirectory(ctx context.Context, destination, alias string) (federation.DirectoryResponse, error) PublicRooms(ctx context.Context, destination string, limit int, since, search string) (json.RawMessage, error) + Hierarchy(ctx context.Context, destination, roomID string, suggestedOnly bool) (json.RawMessage, error) } // federating reports whether this server may talk to others right now. @@ -226,6 +227,8 @@ func storedOf(roomID string, e *event.Event) storage.StoredEvent { if key, ok := e.StateKey(); ok { s.StateKey = &key } + s.RelatesTo, s.RelType = relationOf(e.Content()) + s.SearchKey, s.SearchText = searchableText(e) return s } diff --git a/server/internal/roomserver/search.go b/server/internal/roomserver/search.go new file mode 100644 index 0000000..d3b2fcc --- /dev/null +++ b/server/internal/roomserver/search.go @@ -0,0 +1,122 @@ +package roomserver + +import ( + "context" + "slices" + + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/storage" + "github.com/dariuszjaniak/warp/server/internal/tracing" +) + +// Full-text search: the part of /search that decides where a user may +// look. +// +// The index holds every searchable event of every room on the server, and +// the query against it is the store's. What is decided here is the set of +// rooms, each bounded to what the user may read of it — the same visible +// range a timeline read is clamped to, so a user who left a room finds what +// was said while they were in it and nothing since, and a user who joined a +// `joined` room finds nothing from before they came. + +// SearchRequest is one page of a search. +type SearchRequest struct { + User string + Term string + + // Keys are the fields to match; empty means all three. + Keys []string + + // Rooms and NotRooms narrow the rooms searched, as a filter's rooms + // and not_rooms do. + Rooms []string + NotRooms []string + + // Filter narrows by type and sender. + Filter storage.TimelineFilter + + Order storage.SearchOrder + Offset int + Before int64 + Limit int +} + +// SearchResult is one event a search found. +type SearchResult struct { + Event ClientEvent + + // Position is the event's stream position, which a search by recency + // pages by. + Position int64 + + Rank float64 +} + +// Search runs one page of a search. +func (r *API) Search(ctx context.Context, req SearchRequest) (_ []SearchResult, _ int, err error) { + ctx, span := tracing.Start(ctx, "roomserver.search") + defer func() { tracing.End(span, err) }() + + memberships, err := r.UserMemberships(ctx, req.User, + []string{event.MembershipJoin, event.MembershipLeave, event.MembershipBan}) + if err != nil { + return nil, 0, err + } + + ranges := make([]storage.SearchRange, 0, len(memberships)) + for _, m := range memberships { + if len(req.Rooms) > 0 && !slices.Contains(req.Rooms, m.RoomID) { + continue + } + if slices.Contains(req.NotRooms, m.RoomID) { + continue + } + visible, err := r.VisibleRange(ctx, m.RoomID, req.User) + if err != nil { + return nil, 0, err + } + if !visible.Allowed { + continue + } + ranges = append(ranges, storage.SearchRange{RoomID: m.RoomID, Low: visible.From, High: visible.To}) + } + if len(ranges) == 0 { + return nil, 0, nil + } + + found, total, err := r.store.Search(ctx, storage.SearchRequest{ + Term: req.Term, + Rooms: ranges, + Keys: req.Keys, + Filter: req.Filter, + Order: req.Order, + Offset: req.Offset, + Before: req.Before, + Limit: req.Limit, + }) + if err != nil { + return nil, 0, err + } + + out := make([]SearchResult, 0, len(found)) + for _, f := range found { + room, err := r.store.Room(ctx, f.Event.RoomID) + if err != nil { + return nil, 0, err + } + rules, err := r.rules(room) + if err != nil { + return nil, 0, err + } + e, err := parseEvent(f.Event, rules) + if err != nil { + return nil, 0, err + } + out = append(out, SearchResult{ + Event: r.clientEvent(f.Event, e, f.Event.RoomID), + Position: f.Event.StreamOrdering, + Rank: f.Rank, + }) + } + return out, total, nil +} diff --git a/server/internal/roomserver/sliding.go b/server/internal/roomserver/sliding.go new file mode 100644 index 0000000..4631b89 --- /dev/null +++ b/server/internal/roomserver/sliding.go @@ -0,0 +1,159 @@ +package roomserver + +import ( + "context" + "slices" + "strings" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// The reads sliding sync needs that /sync does not: a room list ordered by +// activity, each room's public face, and pieces of a room's state picked +// tuple by tuple rather than all of it. +// +// A classic sync sends a room's whole state once and then every change; +// sliding sync sends only the tuples the client asked for, which for a room +// list is a name and an avatar out of a state that may hold ten thousand +// member events. Reading the tuples before the event bodies is what makes +// that cheap. + +// bumpEventTypes are the events that move a room up a room list: the +// things a user would call activity. A membership change, a reaction or a +// receipt is not, or a busy room would sit on top of the list for reasons +// nobody looking at it could see. +var bumpEventTypes = []string{ + "m.room.create", + TypeMessage, + "m.room.encrypted", + "m.sticker", + "m.call.invite", + "m.poll.start", + "m.beacon_info", +} + +// BumpStamps returns each room's latest activity at or before to. +func (r *API) BumpStamps(ctx context.Context, roomIDs []string, to int64) (map[string]int64, error) { + return r.store.LatestEventPositions(ctx, roomIDs, bumpEventTypes, to) +} + +// RoomListEntry is what a room list shows of a room. +type RoomListEntry struct { + storage.RoomSummary + + // Encrypted reports an m.room.encryption in the room's state. + Encrypted bool +} + +// RoomListEntry reads one room's public face. +func (r *API) RoomListEntry(ctx context.Context, roomID string) (RoomListEntry, error) { + summary, err := r.store.RoomSummary(ctx, roomID) + if err != nil { + return RoomListEntry{}, err + } + out := RoomListEntry{RoomSummary: summary} + if _, err := r.store.CurrentStateEvent(ctx, roomID, TypeEncryption, ""); err == nil { + out.Encrypted = true + } else if !isNotFound(err) { + return RoomListEntry{}, err + } + return out, nil +} + +// RequiredStateRequest picks pieces of a room's state. +type RequiredStateRequest struct { + RoomID string + User string + + // Since, when set, asks only for the tuples that changed after it: the + // incremental half of a sliding sync. Zero is the state in full. + Since int64 + + // To is the position the state is read at, clamped to what the user + // may see. Zero is the top of the stream. + To int64 + + // Match picks the tuples. + Match func(eventType, stateKey string) bool +} + +// RequiredState reads the tuples of a room's state that Match picks. +func (r *API) RequiredState(ctx context.Context, req RequiredStateRequest) ([]ClientEvent, error) { + room, rules, err := r.roomAndRules(ctx, req.RoomID) + if err != nil { + return nil, err + } + visible, err := r.requireReadable(ctx, room.ID, req.User) + if err != nil { + return nil, err + } + to := req.To + if visible.To != 0 && (to == 0 || to > visible.To) { + to = visible.To + } + + var after []storage.StateEntry + if to == 0 { + after, err = r.store.CurrentState(ctx, req.RoomID) + } else { + after, err = r.store.StateAtStream(ctx, req.RoomID, to) + } + if err != nil { + return nil, err + } + + var before map[[2]string]string + if req.Since != 0 { + entries, err := r.store.StateAtStream(ctx, req.RoomID, req.Since) + if err != nil { + return nil, err + } + before = make(map[[2]string]string, len(entries)) + for _, e := range entries { + before[[2]string{e.Type, e.StateKey}] = e.EventID + } + } + + var ids []string + for _, e := range after { + if !req.Match(e.Type, e.StateKey) { + continue + } + if before != nil && before[[2]string{e.Type, e.StateKey}] == e.EventID { + continue + } + ids = append(ids, e.EventID) + } + if len(ids) == 0 { + return []ClientEvent{}, nil + } + events, err := r.clientEventsByID(ctx, req.RoomID, rules, ids) + if err != nil { + return nil, err + } + slices.SortFunc(events, func(a, b ClientEvent) int { + if c := strings.Compare(a.Type, b.Type); c != 0 { + return c + } + return strings.Compare(*a.StateKey, *b.StateKey) + }) + return events, nil +} + +// SpaceChildRooms lists the rooms a space this server is in names as its +// children. +func (r *API) SpaceChildRooms(ctx context.Context, spaceID string) ([]string, error) { + local, err := r.participating(ctx, spaceID) + if err != nil || !local { + return nil, err + } + read, err := r.readSpaceRoom(ctx, spaceID, false) + if err != nil { + return nil, err + } + out := make([]string, 0, len(read.children)) + for _, c := range read.children { + out = append(out, c.roomID) + } + return out, nil +} diff --git a/server/internal/roomserver/stream.go b/server/internal/roomserver/stream.go index 3590aa5..d972711 100644 --- a/server/internal/roomserver/stream.go +++ b/server/internal/roomserver/stream.go @@ -215,6 +215,9 @@ func (r *API) timeline(ctx context.Context, rules roomversion.Rules, visible Vis } out.Events = append(out.Events, r.clientEvent(s, e, req.RoomID)) } + if err := r.bundleAggregations(ctx, req.RoomID, rules, req.User, visible, stored, out.Events); err != nil { + return Timeline{}, err + } // Start sits below the oldest event, so that handing it back as a High // bound returns the page before this one; End is at the newest, so that // handing it back as a Low bound returns the page after. Below the diff --git a/server/internal/setup/observability.go b/server/internal/setup/observability.go new file mode 100644 index 0000000..63cf7a5 --- /dev/null +++ b/server/internal/setup/observability.go @@ -0,0 +1,90 @@ +package setup + +import ( + "context" + "database/sql" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + + "github.com/dariuszjaniak/warp/server/internal/metrics" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/syncapi" +) + +// The homeserver's own metrics, beside the HTTP and runtime ones every +// route gets: what a dashboard (deploy/grafana) shows about a server's +// rooms rather than its requests. +// +// They are gauges read on scrape and one counter fed from the roomserver's +// emit stage, so that none of them costs anything between scrapes. + +// eventCounter counts persisted events from the emit stage. +type eventCounter struct { + events *prometheus.CounterVec +} + +// EventPersisted implements roomserver.Notifier. +func (c *eventCounter) EventPersisted(_ context.Context, n roomserver.Notification) { + origin := "local" + if n.Origin != "" { + origin = "federation" + } + kind := "message" + if n.Event != nil && n.Event.IsState() { + kind = "state" + } + c.events.WithLabelValues(origin, kind).Inc() +} + +// registerServerMetrics adds the homeserver's collectors to the registry, +// and returns the event counter to put on the emit stage. +func (s *Server) registerServerMetrics(notifier *syncapi.Notifier) *eventCounter { + counter := &eventCounter{events: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metrics.Namespace, + Name: "events_persisted_total", + Help: "Room events persisted, by where they came from and whether they are state.", + }, []string{"origin", "kind"})} + s.metrics.MustRegister(counter.events) + + s.metrics.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: metrics.Namespace, + Subsystem: "sync", + Name: "waiting", + Help: "Syncs blocked in a long poll, classic and sliding.", + }, func() float64 { return float64(notifier.Listeners()) })) + + s.metrics.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: metrics.Namespace, + Subsystem: "sync", + Name: "sliding_connections", + Help: "Sliding sync connections whose state the server is holding.", + }, func() float64 { + if s.sync == nil { + return 0 + } + return float64(s.sync.SlidingConnections()) + })) + + s.metrics.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: metrics.Namespace, + Name: "federation_enabled", + Help: "1 while federation is switched on.", + }, func() float64 { + if s.settings != nil && s.settings.FederationEnabled() { + return 1 + } + return 0 + })) + return counter +} + +// registerDBMetrics adds the connection pool's statistics, for a driver +// that is built on database/sql. +func (s *Server) registerDBMetrics() { + pool, ok := s.db.(interface{ DB() *sql.DB }) + if !ok { + return + } + s.metrics.MustRegister(collectors.NewDBStatsCollector(pool.DB(), s.db.Driver())) +} diff --git a/server/internal/setup/phase10_test.go b/server/internal/setup/phase10_test.go new file mode 100644 index 0000000..893e84f --- /dev/null +++ b/server/internal/setup/phase10_test.go @@ -0,0 +1,170 @@ +package setup_test + +import ( + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/dariuszjaniak/warp/server/internal/config" +) + +// The Phase 10 tests: the PocketBase driver and the new endpoints, through +// the whole stack. + +// TestPocketBaseServer runs a server on the pocketbase driver: accounts, +// rooms, search and media, with the media landing in PocketBase's own +// filesystem rather than under media.base_path. +func TestPocketBaseServer(t *testing.T) { + dir := t.TempDir() + dataDir := filepath.Join(dir, "pb_data") + mediaDir := filepath.Join(dir, "media") + srv := newSigningServer(t, func(cfg *config.Config) { + cfg.Database.Driver = "pocketbase" + cfg.Database.ConnectionString = dataDir + cfg.Registration.Enabled = true + cfg.ClientAPI.RateLimit.Enabled = false + cfg.Media.BasePath = mediaDir + }) + + health := get(t, srv, http.MethodGet, "/healthz") + if health.Code != http.StatusOK || !strings.Contains(health.Body.String(), `"database":"pocketbase"`) { + t.Fatalf("healthz: %d %s", health.Code, health.Body.String()) + } + + token := registerThrough(t, srv, "alice") + rec := post(t, srv, "/_matrix/client/v3/createRoom", map[string]any{"name": "On PocketBase"}, token) + if rec.Code != http.StatusOK { + t.Fatalf("createRoom: %d %s", rec.Code, rec.Body.String()) + } + roomID := decode[map[string]string](t, rec)["room_id"] + + rec = send(t, srv, http.MethodPut, + "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/send/m.room.message/t1", + "application/json", []byte(`{"msgtype":"m.text","body":"stored in a pocket"}`), token, nil) + if rec.Code != http.StatusOK { + t.Fatalf("send: %d %s", rec.Code, rec.Body.String()) + } + + rec = post(t, srv, "/_matrix/client/v3/search", map[string]any{ + "search_categories": map[string]any{"room_events": map[string]any{"search_term": "pocket"}}, + }, token) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "stored in a pocket") { + t.Errorf("search: %d %s", rec.Code, rec.Body.String()) + } + + server, id := uploadThrough(t, srv, token, "text/plain", "note.txt", []byte("pocket media")) + rec = getWithToken(t, srv, "/_matrix/client/v1/media/download/"+server+"/"+id, token) + if rec.Code != http.StatusOK || rec.Body.String() != "pocket media" { + t.Fatalf("download: %d %q", rec.Code, rec.Body.String()) + } + + var found bool + _ = filepath.WalkDir(filepath.Join(dataDir, "storage", "warp_media"), func(_ string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() { + found = true + } + return nil + }) + if !found { + t.Error("the upload is not in PocketBase's storage directory") + } + if _, err := os.Stat(mediaDir); !os.IsNotExist(err) { + t.Errorf("media.base_path was used: %v", err) + } +} + +func TestVersionsAdvertisesSlidingSync(t *testing.T) { + srv := newTestServer(t, nil) + rec := get(t, srv, http.MethodGet, "/_matrix/client/versions") + features := decode[map[string]any](t, rec)["unstable_features"].(map[string]any) + if features["org.matrix.simplified_msc3575"] != true { + t.Errorf("unstable_features = %v", features) + } +} + +func TestPhase10RoutesAreWiredUp(t *testing.T) { + srv := newSigningServer(t, func(c *config.Config) { + c.Registration.Enabled = true + c.ClientAPI.RateLimit.Enabled = false + }) + token := registerThrough(t, srv, "alice") + rec := post(t, srv, "/_matrix/client/v3/createRoom", map[string]any{ + "creation_content": map[string]any{"type": "m.space"}, + }, token) + roomID := decode[map[string]string](t, rec)["room_id"] + escaped := url.PathEscape(roomID) + + for _, path := range []string{ + "/_matrix/client/v1/rooms/" + escaped + "/hierarchy", + "/_matrix/client/v1/rooms/" + escaped + "/threads", + "/_matrix/client/v1/room_summary/" + escaped, + } { + if rec := getWithToken(t, srv, path, token); rec.Code != http.StatusOK { + t.Errorf("%s: %d %s", path, rec.Code, rec.Body.String()) + } + } + rec = post(t, srv, "/_matrix/client/unstable/org.matrix.simplified_msc3575/sync", map[string]any{ + "lists": map[string]any{"all": map[string]any{"ranges": [][2]int{{0, 5}}}}, + }, token) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"pos"`) { + t.Errorf("sliding sync: %d %s", rec.Code, rec.Body.String()) + } +} + +func TestServerMetrics(t *testing.T) { + srv := newSigningServer(t, nil) + token := registerThrough(t, srv, "alice") + if rec := post(t, srv, "/_matrix/client/v3/createRoom", map[string]any{}, token); rec.Code != http.StatusOK { + t.Fatalf("createRoom: %d", rec.Code) + } + body := get(t, srv, http.MethodGet, "/metrics").Body.String() + for _, want := range []string{ + `warp_events_persisted_total{kind="state",origin="local"}`, + "warp_sync_waiting 0", + "warp_sync_sliding_connections 0", + "warp_federation_enabled 0", + "go_sql_open_connections", + } { + if !strings.Contains(body, want) { + t.Errorf("/metrics has no %s", want) + } + } +} + +// TestIdleSyncLongPolls is the regression test for two ways an idle +// /sync came back at once instead of waiting, which made every client's +// long poll a busy loop: push rules sent again on every sync of a server +// with no account data written yet, and an all-zero token taken for a +// first sync on a server whose streams had not moved at all. +func TestIdleSyncLongPolls(t *testing.T) { + for _, withRoom := range []bool{false, true} { + t.Run(map[bool]string{false: "empty server", true: "with a room"}[withRoom], func(t *testing.T) { + srv := newSigningServer(t, nil) + token := registerThrough(t, srv, "alice") + if withRoom { + if rec := post(t, srv, "/_matrix/client/v3/createRoom", map[string]any{}, token); rec.Code != http.StatusOK { + t.Fatalf("createRoom: %d", rec.Code) + } + } + + first := decode[map[string]any](t, getWithToken(t, srv, "/_matrix/client/v3/sync", token)) + since, _ := first["next_batch"].(string) + + body := decode[map[string]any](t, getWithToken(t, srv, "/_matrix/client/v3/sync?timeout=0&since="+url.QueryEscape(since), token)) + accountData, _ := body["account_data"].(map[string]any) + if events, _ := accountData["events"].([]any); len(events) != 0 { + t.Errorf("an idle incremental sync sent account data again: %d events", len(events)) + } + + start := time.Now() + getWithToken(t, srv, "/_matrix/client/v3/sync?timeout=1000&since="+url.QueryEscape(since), token) + if waited := time.Since(start); waited < 900*time.Millisecond { + t.Errorf("an idle sync returned after %s instead of waiting out its timeout", waited) + } + }) + } +} diff --git a/server/internal/setup/routes.go b/server/internal/setup/routes.go index 4a11bbb..829c9e3 100644 --- a/server/internal/setup/routes.go +++ b/server/internal/setup/routes.go @@ -6,6 +6,7 @@ import ( "github.com/dariuszjaniak/warp/server/internal/buildinfo" "github.com/dariuszjaniak/warp/server/internal/httputil" + "github.com/dariuszjaniak/warp/server/internal/tracing" ) // SpecVersions is what GET /_matrix/client/versions reports. @@ -20,8 +21,11 @@ import ( var SpecVersions = []string{"v1.13", "v1.14"} // UnstableFeatures advertises MSC support. Simplified Sliding Sync -// (MSC4186) is Phase 10. -var UnstableFeatures = map[string]bool{} +// (MSC4186) is how Element X and the matrix-rust-sdk find out they can +// skip /sync. +var UnstableFeatures = map[string]bool{ + "org.matrix.simplified_msc3575": true, +} type versionsResponse struct { Versions []string `json:"versions"` @@ -51,7 +55,12 @@ type healthResponse struct { // routes registers every endpoint this server serves. func (s *Server) routes() *httputil.Router { - r := httputil.NewRouter(s.metrics.Instrument) + // Every route is counted and traced under its pattern. The span is + // inside the metrics wrapper so that what the histogram times is what + // the trace shows. + r := httputil.NewRouter(func(route string, h http.Handler) http.Handler { + return s.metrics.Instrument(route, tracing.Instrument(route, h)) + }) r.HandleFunc("GET /_matrix/client/versions", s.handleVersions) r.HandleFunc("GET /.well-known/matrix/client", s.handleWellKnownClient) diff --git a/server/internal/setup/server.go b/server/internal/setup/server.go index e6d2453..7d4ef42 100644 --- a/server/internal/setup/server.go +++ b/server/internal/setup/server.go @@ -33,6 +33,7 @@ import ( "github.com/dariuszjaniak/warp/server/internal/storage" _ "github.com/dariuszjaniak/warp/server/internal/storage/drivers" // link in the storage drivers "github.com/dariuszjaniak/warp/server/internal/syncapi" + "github.com/dariuszjaniak/warp/server/internal/tracing" "github.com/dariuszjaniak/warp/server/internal/userapi" ) @@ -47,6 +48,10 @@ type Server struct { federation *federation.Gate signingKey *signing.KeyPair + // stopTracing flushes and stops the tracer provider tracing.Setup + // installed. + stopTracing func(context.Context) error + users *userapi.API rooms *roomserver.API sync *syncapi.API @@ -122,21 +127,29 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) s := &Server{cfg: cfg, log: log, metrics: metrics.NewRegistry()} + stopTracing, err := tracing.Setup(ctx, tracing.Config{ + Enabled: cfg.Tracing.Enabled, + Endpoint: cfg.Tracing.Endpoint, + SampleRatio: cfg.Tracing.SampleRatio, + ServiceName: cfg.Tracing.ServiceName, + ServiceVersion: buildinfo.Version, + ServerName: cfg.Global.ServerName, + }) + if err != nil { + return nil, err + } + s.stopTracing = stopTracing + if opts.Database != nil { s.db = opts.Database } else { - db, err := storage.Open(ctx, storage.Config{ - Driver: cfg.Database.Driver, - ConnectionString: cfg.Database.ConnectionString, - MaxOpenConns: cfg.Database.MaxOpenConns, - MaxIdleConns: cfg.Database.MaxIdleConns, - ConnMaxLifetime: cfg.Database.ConnMaxLifetime, - }, log) + db, err := storage.Open(ctx, cfg.Database.Storage(), log) if err != nil { return nil, err } s.db, s.ownsDB = db, true } + s.registerDBMetrics() if err := s.prepareSchema(ctx); err != nil { _ = s.closeDB() @@ -258,6 +271,7 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) notifier := syncapi.NewNotifier() emit := &fanout{} emit.add(notifier) + emit.add(s.registerServerMetrics(notifier)) clientTLS := opts.FederationTLS if clientTLS == nil { @@ -734,6 +748,14 @@ func (s *Server) Close() error { // cursors. s.appservices.Close() } + if s.stopTracing != nil { + // Spans still in the batch are sent rather than lost. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.stopTracing(ctx); err != nil { + s.log.Warn("could not flush traces", "error", err) + } + } return s.closeDB() } diff --git a/server/internal/setup/threads_test.go b/server/internal/setup/threads_test.go new file mode 100644 index 0000000..51ace79 --- /dev/null +++ b/server/internal/setup/threads_test.go @@ -0,0 +1,121 @@ +package setup_test + +import ( + "net/http" + "net/url" + "testing" + + "github.com/dariuszjaniak/warp/server/internal/setup" +) + +// threadCounts is the part of a sync the per-thread count test reads. +type threadCounts struct { + NotificationCount int `json:"notification_count"` + HighlightCount int `json:"highlight_count"` +} + +type threadSync struct { + Rooms struct { + Join map[string]struct { + UnreadNotifications *threadCounts `json:"unread_notifications"` + UnreadThreadNotifications map[string]threadCounts `json:"unread_thread_notifications"` + } `json:"join"` + } `json:"rooms"` +} + +// threadSyncOnce is an initial sync, with the per-thread counts asked for +// or not. +func threadSyncOnce(t *testing.T, srv *setup.Server, token string, byThread bool) threadSync { + t.Helper() + path := "/_matrix/client/v3/sync?timeout=0" + if byThread { + path += "&filter=" + url.QueryEscape(`{"room":{"timeline":{"unread_thread_notifications":true}}}`) + } + rec := getWithToken(t, srv, path, token) + if rec.Code != http.StatusOK { + t.Fatalf("sync: %d %s", rec.Code, rec.Body.String()) + } + return decode[threadSync](t, rec) +} + +// TestThreadNotificationCountsThroughTheAPI follows notifications into a +// thread and out again: counted under the thread with the filter flag and +// in the room total without it, and read by a thread receipt, a main +// receipt and an unthreaded one, each clearing only its own part. +func TestThreadNotificationCountsThroughTheAPI(t *testing.T) { + srv := newSigningServer(t, nil) + alice := registerThrough(t, srv, "alice") + bob := registerThrough(t, srv, "bob") + + roomID := createPublicRoom(t, srv, alice) + mustOK(t, post(t, srv, "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/join", map[string]any{}, bob), "join") + waitForPush(t, srv) + + inThread := func(root string) map[string]any { + return map[string]any{"rel_type": "m.thread", "event_id": root} + } + root := sendText(t, srv, bob, roomID, "r", map[string]any{"msgtype": "m.text", "body": "a thread"}) + sendText(t, srv, bob, roomID, "a", map[string]any{"msgtype": "m.text", "body": "first reply", + "m.relates_to": inThread(root)}) + reply := sendText(t, srv, bob, roomID, "b", map[string]any{"msgtype": "m.text", "body": "second reply", + "m.mentions": map[string]any{"user_ids": []string{"@alice:warp.example"}}, + "m.relates_to": inThread(root)}) + sendText(t, srv, bob, roomID, "c", map[string]any{"msgtype": "m.text", "body": "back in the room"}) + waitForPush(t, srv) + + // Without the flag the room counts everything, as before threads. + plain := threadSyncOnce(t, srv, alice, false).Rooms.Join[roomID] + if c := plain.UnreadNotifications; c == nil || *c != (threadCounts{NotificationCount: 4, HighlightCount: 1}) { + t.Errorf("room total = %+v", c) + } + if plain.UnreadThreadNotifications != nil { + t.Errorf("thread counts without the flag: %+v", plain.UnreadThreadNotifications) + } + + // With it the root and the later message are the main timeline's, and + // the replies are the thread's. + split := threadSyncOnce(t, srv, alice, true).Rooms.Join[roomID] + if c := split.UnreadNotifications; c == nil || *c != (threadCounts{NotificationCount: 2}) { + t.Errorf("main timeline = %+v", c) + } + if got := split.UnreadThreadNotifications; len(got) != 1 || + got[root] != (threadCounts{NotificationCount: 2, HighlightCount: 1}) { + t.Errorf("thread counts = %+v", got) + } + + receipt := func(eventID string, body map[string]any) { + t.Helper() + mustOK(t, post(t, srv, "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/receipt/m.read/"+url.PathEscape(eventID), + body, alice), "receipt") + } + + // Reading the thread leaves the main timeline unread. + receipt(reply, map[string]any{"thread_id": root}) + split = threadSyncOnce(t, srv, alice, true).Rooms.Join[roomID] + if c := split.UnreadNotifications; c == nil || c.NotificationCount != 2 || len(split.UnreadThreadNotifications) != 0 { + t.Errorf("after the thread receipt: %+v, %+v", c, split.UnreadThreadNotifications) + } + if c := threadSyncOnce(t, srv, alice, false).Rooms.Join[roomID].UnreadNotifications; c == nil || c.NotificationCount != 2 { + t.Errorf("room total after the thread receipt = %+v", c) + } + + // A new reply, then a main receipt on the root: the root is read, the + // later message and the new reply are not. + third := sendText(t, srv, bob, roomID, "d", map[string]any{"msgtype": "m.text", "body": "third reply", + "m.relates_to": inThread(root)}) + waitForPush(t, srv) + receipt(root, map[string]any{"thread_id": "main"}) + split = threadSyncOnce(t, srv, alice, true).Rooms.Join[roomID] + if c := split.UnreadNotifications; c == nil || c.NotificationCount != 1 || + split.UnreadThreadNotifications[root].NotificationCount != 1 { + t.Errorf("after the main receipt: %+v, %+v", c, split.UnreadThreadNotifications) + } + + // An unthreaded receipt reads the lot up to its event, whichever + // thread that event is in: here the newest, a reply. + receipt(third, map[string]any{}) + split = threadSyncOnce(t, srv, alice, true).Rooms.Join[roomID] + if c := split.UnreadNotifications; c == nil || c.NotificationCount != 0 || len(split.UnreadThreadNotifications) != 0 { + t.Errorf("after the unthreaded receipt: %+v, %+v", c, split.UnreadThreadNotifications) + } +} diff --git a/server/internal/storage/pocketbase/pocketbase.go b/server/internal/storage/pocketbase/pocketbase.go index 67cc1fc..27341a4 100644 --- a/server/internal/storage/pocketbase/pocketbase.go +++ b/server/internal/storage/pocketbase/pocketbase.go @@ -1,750 +1,278 @@ -// Package pocketbase is the placeholder for the PocketBase storage driver. +// Package pocketbase is the PocketBase storage driver: Warp on an embedded +// PocketBase app, for "appliance" deployments that want one binary and one +// data directory. // -// The plan (PLAN.md section 4) is an appliance deployment: one binary that -// also brings admin auth, a filesystem abstraction for media (local or S3) -// and realtime for the admin panel. The event DAG is a poor fit for -// PocketBase's record model, so the roomserver and syncapi tables will use -// its underlying dbx connection directly rather than its collection API. +// PLAN.md section 4 set out the shape, and open question 4 asked how far +// to take it. The answer here is the cheaper of the two: the event DAG and +// everything else Warp stores are a poor fit for PocketBase's record +// model, so the Warp tables are the sqlite driver's own, migrated by Warp +// and written through the same shared store, on PocketBase's underlying +// SQLite connection rather than through its collection API. What PocketBase +// adds is everything around that: its data directory and its system +// tables, which a PocketBase admin or a later Warp feature can build on, and +// its filesystem abstraction, which puts media content on local disk or in +// an S3 bucket. // -// Until that work starts (Phase 10) the driver is compiled in, registered -// and fails loudly, so that `database.driver: pocketbase` is a clear error -// instead of an unknown-driver one, and so the interface stays honest: -// every method the other drivers implement exists here too. +// The result is at parity with the sqlite driver by construction — it is +// the same SQL on the same engine — and the storage tests run against it +// to keep it that way. package pocketbase import ( "context" + "crypto/sha256" + "encoding/hex" + "errors" "fmt" + "io" "log/slog" + "os" + "path/filepath" + + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase/core" + _ "github.com/pocketbase/pocketbase/migrations" // PocketBase's system migrations; without them its bootstrap has no tables + "github.com/pocketbase/pocketbase/tools/filesystem" "github.com/dariuszjaniak/warp/server/internal/storage" + "github.com/dariuszjaniak/warp/server/internal/storage/sqlite" + "github.com/dariuszjaniak/warp/server/internal/storage/sqlstore" ) func init() { storage.Register(storage.DriverPocketBase, Open) } -// Database is the not-yet-implemented PocketBase driver. -type Database struct{} - -// Open always fails with storage.ErrNotImplemented. -func Open(context.Context, storage.Config, *slog.Logger) (storage.Database, error) { - return nil, fmt.Errorf( - "pocketbase driver is not implemented yet (planned for Phase 10); use %q or %q: %w", - storage.DriverSQLite, storage.DriverPostgres, storage.ErrNotImplemented) -} - -// Driver implements storage.Database. -func (*Database) Driver() string { return storage.DriverPocketBase } - -// Ping implements storage.Database. -func (*Database) Ping(context.Context) error { return storage.ErrNotImplemented } - -// Migrate implements storage.Database. -func (*Database) Migrate(context.Context) error { return storage.ErrNotImplemented } - -// PendingMigrations implements storage.Database. -func (*Database) PendingMigrations(context.Context) ([]string, error) { - return nil, storage.ErrNotImplemented -} - -// Settings implements storage.Database. -func (*Database) Settings() storage.SettingsStore { return notImplementedSettings{} } - -// Users implements storage.Database. -func (*Database) Users() storage.UserStore { return notImplementedUsers{} } - -// Rooms implements storage.Database. -func (*Database) Rooms() storage.RoomStore { return notImplementedRooms{} } - -// Sync implements storage.Database. -func (*Database) Sync() storage.SyncStore { return notImplementedSync{} } - -// Federation implements storage.Database. -func (*Database) Federation() storage.FederationStore { return notImplementedFederation{} } - -// Media implements storage.Database. -func (*Database) Media() storage.MediaStore { return notImplementedMedia{} } - -// Keys implements storage.Database. -func (*Database) Keys() storage.KeyStore { return notImplementedKeys{} } - -// Push implements storage.Database. -func (*Database) Push() storage.PushStore { return notImplementedPush{} } - -// Admin implements storage.Database. -func (*Database) Admin() storage.AdminStore { return notImplementedAdmin{} } - -// AppServices implements storage.Database. -func (*Database) AppServices() storage.AppServiceStore { return notImplementedAppServices{} } - -// Close implements storage.Database. -func (*Database) Close() error { return nil } - -// notImplementedSettings satisfies storage.SettingsStore by refusing every -// call, so a partially wired-up PocketBase deployment cannot silently lose -// an admin's federation switch. -type notImplementedSettings struct{} - -func (notImplementedSettings) Get(context.Context, string) (string, bool, error) { - return "", false, storage.ErrNotImplemented -} - -func (notImplementedSettings) GetAll(context.Context) (map[string]string, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedSettings) Set(context.Context, string, string) error { - return storage.ErrNotImplemented -} - -func (notImplementedSettings) Delete(context.Context, string) error { - return storage.ErrNotImplemented -} - -// notImplementedUsers satisfies storage.UserStore the same way: a -// half-wired PocketBase deployment must not accept a registration it cannot -// store, or hand out an access token it cannot look up again. -type notImplementedUsers struct{} - -func (notImplementedUsers) CreateAccount(context.Context, storage.NewAccount) (storage.Account, error) { - return storage.Account{}, storage.ErrNotImplemented -} - -func (notImplementedUsers) Account(context.Context, string) (storage.Account, error) { - return storage.Account{}, storage.ErrNotImplemented -} - -func (notImplementedUsers) PasswordHash(context.Context, string) (string, error) { - return "", storage.ErrNotImplemented -} - -func (notImplementedUsers) SetPasswordHash(context.Context, string, string) error { - return storage.ErrNotImplemented -} - -func (notImplementedUsers) SetAdmin(context.Context, string, bool) error { - return storage.ErrNotImplemented -} - -func (notImplementedUsers) SetDisplayName(context.Context, string, string) error { - return storage.ErrNotImplemented -} - -func (notImplementedUsers) SetDeactivated(context.Context, string, bool) error { - return storage.ErrNotImplemented -} - -func (notImplementedUsers) SetAvatarURL(context.Context, string, string) error { - return storage.ErrNotImplemented -} - -func (notImplementedUsers) ListAccounts(context.Context, storage.AccountFilter) ([]storage.Account, int, error) { - return nil, 0, storage.ErrNotImplemented -} - -func (notImplementedUsers) Stats(context.Context) (storage.Stats, error) { - return storage.Stats{}, storage.ErrNotImplemented -} - -func (notImplementedUsers) CreateSession(context.Context, storage.NewSession) (storage.Device, error) { - return storage.Device{}, storage.ErrNotImplemented -} - -func (notImplementedUsers) RefreshSession(context.Context, storage.RefreshedSession) (storage.TokenInfo, error) { - return storage.TokenInfo{}, storage.ErrNotImplemented -} - -func (notImplementedUsers) TokenInfo(context.Context, string) (storage.TokenInfo, error) { - return storage.TokenInfo{}, storage.ErrNotImplemented -} - -func (notImplementedUsers) TouchDevice(context.Context, string, string, string, string, int64) error { - return storage.ErrNotImplemented -} - -func (notImplementedUsers) Device(context.Context, string, string) (storage.Device, error) { - return storage.Device{}, storage.ErrNotImplemented -} - -func (notImplementedUsers) Devices(context.Context, string) ([]storage.Device, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedUsers) SetDeviceDisplayName(context.Context, string, string, string) error { - return storage.ErrNotImplemented -} - -func (notImplementedUsers) DeleteDevices(context.Context, string, []string) (int, error) { - return 0, storage.ErrNotImplemented -} - -func (notImplementedUsers) DeleteAllDevices(context.Context, string) (int, error) { - return 0, storage.ErrNotImplemented -} - -func (notImplementedUsers) AccountData(context.Context, string, string, string) ([]byte, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedUsers) SetAccountData(context.Context, string, string, string, []byte) error { - return storage.ErrNotImplemented -} - -// notImplementedRooms satisfies storage.RoomStore the same way. It matters -// more here than anywhere else: a room whose create event was accepted and -// not stored would hand a client a room ID that refers to nothing. -type notImplementedRooms struct{} - -func (notImplementedRooms) CreateRoom(context.Context, storage.NewRoom) error { - return storage.ErrNotImplemented -} - -func (notImplementedRooms) Room(context.Context, string) (storage.Room, error) { - return storage.Room{}, storage.ErrNotImplemented -} - -func (notImplementedRooms) Write(context.Context, storage.RoomWrite) (int64, error) { - return 0, storage.ErrNotImplemented -} - -func (notImplementedRooms) Event(context.Context, string, string) (storage.StoredEvent, error) { - return storage.StoredEvent{}, storage.ErrNotImplemented -} - -func (notImplementedRooms) EventsByID(context.Context, string, []string) ([]storage.StoredEvent, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) CurrentState(context.Context, string) ([]storage.StateEntry, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) CurrentStateEvents(context.Context, string) ([]storage.StoredEvent, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) CurrentStateEvent(context.Context, string, string, string) (storage.StoredEvent, error) { - return storage.StoredEvent{}, storage.ErrNotImplemented -} - -func (notImplementedRooms) StateBefore(context.Context, string, string) ([]storage.StateEntry, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) Extremities(context.Context, string) ([]storage.StoredEvent, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) Members(context.Context, string, []string) ([]storage.StoredEvent, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) MembershipEntries(context.Context, string, []string) ([]storage.MembershipEntry, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) Membership(context.Context, string, string) (storage.MembershipEntry, error) { - return storage.MembershipEntry{}, storage.ErrNotImplemented -} - -func (notImplementedRooms) RoomsForUser(context.Context, string, []string) ([]string, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) MembershipsForUser(context.Context, string, []string) ([]storage.SyncMembership, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) UserMembershipEvents(context.Context, string, string) ([]storage.StoredEvent, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) MaxStreamOrdering(context.Context) (int64, error) { - return 0, storage.ErrNotImplemented -} - -func (notImplementedRooms) StreamEvents(context.Context, int64, int64, int) ([]storage.StoredEvent, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) Timeline(context.Context, storage.TimelineRequest) ([]storage.StoredEvent, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) RoomsWithEvents(context.Context, []string, int64, int64) ([]string, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) StateAtStream(context.Context, string, int64) ([]storage.StateEntry, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) TransactionEvent(context.Context, storage.TransactionRef) (string, error) { - return "", storage.ErrNotImplemented -} - -func (notImplementedRooms) CreateAlias(context.Context, storage.RoomAlias) error { - return storage.ErrNotImplemented -} - -func (notImplementedRooms) DeleteAlias(context.Context, string) error { - return storage.ErrNotImplemented -} - -func (notImplementedRooms) Alias(context.Context, string) (storage.RoomAlias, error) { - return storage.RoomAlias{}, storage.ErrNotImplemented -} - -func (notImplementedRooms) AliasesForRoom(context.Context, string) ([]string, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) SetPublished(context.Context, string, bool) error { - return storage.ErrNotImplemented -} - -func (notImplementedRooms) PublishedRooms(context.Context, storage.RoomFilter) ([]storage.RoomSummary, int, error) { - return nil, 0, storage.ErrNotImplemented -} - -func (notImplementedRooms) ListRooms(context.Context, storage.RoomFilter) ([]storage.RoomSummary, int, error) { - return nil, 0, storage.ErrNotImplemented -} - -func (notImplementedRooms) RoomSummary(context.Context, string) (storage.RoomSummary, error) { - return storage.RoomSummary{}, storage.ErrNotImplemented -} - -func (notImplementedRooms) PurgeRoom(context.Context, string) error { - return storage.ErrNotImplemented -} - -func (notImplementedRooms) RoomStats(context.Context) (storage.RoomStats, error) { - return storage.RoomStats{}, storage.ErrNotImplemented -} - -func (notImplementedRooms) WriteOutliers(context.Context, string, []storage.StoredEvent) error { - return storage.ErrNotImplemented -} - -func (notImplementedRooms) WriteBackfill(context.Context, string, []storage.BackfillEvent) error { - return storage.ErrNotImplemented -} - -func (notImplementedRooms) InviteState(context.Context, string, string) ([]byte, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) HasJoinedMemberOn(context.Context, string, string) (bool, error) { - return false, storage.ErrNotImplemented -} - -func (notImplementedRooms) EventByID(context.Context, string) (storage.StoredEvent, error) { - return storage.StoredEvent{}, storage.ErrNotImplemented -} - -func (notImplementedRooms) UsersSharingRooms(context.Context, string) ([]string, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedRooms) MemberChanges(context.Context, []string, int64, int64) ([]storage.MemberChange, error) { - return nil, storage.ErrNotImplemented -} - -// notImplementedFederation satisfies storage.FederationStore the same way. -// A queue that accepted events and never stored them would lose them the -// moment federation was switched on. -type notImplementedFederation struct{} - -func (notImplementedFederation) StoreServerKeys(context.Context, []storage.ServerKey) error { - return storage.ErrNotImplemented -} - -func (notImplementedFederation) ServerKeys(context.Context, string, []string) ([]storage.ServerKey, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedFederation) Enqueue(context.Context, []storage.QueuedItem) ([]storage.QueuedItem, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedFederation) Queued(context.Context, string, string, int) ([]storage.QueuedItem, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedFederation) Dequeue(context.Context, []int64) error { - return storage.ErrNotImplemented -} - -func (notImplementedFederation) QueuedDestinations(context.Context) ([]string, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedFederation) Destination(context.Context, string) (storage.Destination, error) { - return storage.Destination{}, storage.ErrNotImplemented -} - -func (notImplementedFederation) SetDestination(context.Context, storage.Destination) error { - return storage.ErrNotImplemented -} - -func (notImplementedFederation) Destinations(context.Context) ([]storage.DestinationSummary, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedFederation) InboundTransaction(context.Context, string, string) ([]byte, bool, error) { - return nil, false, storage.ErrNotImplemented -} - -func (notImplementedFederation) RecordInboundTransaction(context.Context, string, string, []byte, int64) error { - return storage.ErrNotImplemented -} - -// notImplementedSync satisfies storage.SyncStore the same way. A sync that -// returned an empty success would be worse than an error: a client would -// take it for "nothing has happened" and stop asking. -type notImplementedSync struct{} - -func (notImplementedSync) StreamPositions(context.Context) (storage.StreamPositions, error) { - return storage.StreamPositions{}, storage.ErrNotImplemented -} - -func (notImplementedSync) SetReceipt(context.Context, storage.Receipt) (int64, bool, error) { - return 0, false, storage.ErrNotImplemented -} - -func (notImplementedSync) Receipts(context.Context, []string, int64, int64) ([]storage.Receipt, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedSync) ReceiptsForUser(context.Context, string, []string) ([]storage.Receipt, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedSync) SendToDevice(context.Context, []storage.NewToDeviceMessage) (int64, error) { - return 0, storage.ErrNotImplemented -} - -func (notImplementedSync) ToDeviceMessages(context.Context, string, string, int64, int64, int) ([]storage.ToDeviceMessage, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedSync) DeleteToDeviceMessages(context.Context, string, string, int64) error { - return storage.ErrNotImplemented -} - -func (notImplementedSync) SetPresence(context.Context, storage.PresenceState) (int64, bool, error) { - return 0, false, storage.ErrNotImplemented -} - -func (notImplementedSync) Presence(context.Context, string) (storage.PresenceState, error) { - return storage.PresenceState{}, storage.ErrNotImplemented -} - -func (notImplementedSync) PresenceForUsers(context.Context, []string, int64, int64) ([]storage.PresenceState, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedSync) AccountDataChanges(context.Context, string, int64, int64) ([]storage.AccountDataEntry, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedSync) AccountDataAt(context.Context, string, int64) ([]storage.AccountDataEntry, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedSync) SaveFilter(context.Context, string, []byte, int64) (string, error) { - return "", storage.ErrNotImplemented -} - -func (notImplementedSync) Filter(context.Context, string, string) ([]byte, error) { - return nil, storage.ErrNotImplemented -} - -// notImplementedMedia satisfies storage.MediaStore the same way. An upload -// whose metadata was not stored would hand the client an mxc:// URI that -// refers to nothing. -type notImplementedMedia struct{} - -func (notImplementedMedia) CreateMedia(context.Context, storage.Media) error { - return storage.ErrNotImplemented -} - -func (notImplementedMedia) Media(context.Context, string, string) (storage.Media, error) { - return storage.Media{}, storage.ErrNotImplemented -} - -func (notImplementedMedia) CompleteUpload(context.Context, string, string, storage.MediaContent) error { - return storage.ErrNotImplemented -} - -func (notImplementedMedia) PendingUploads(context.Context, string, int64) (int, error) { - return 0, storage.ErrNotImplemented -} - -func (notImplementedMedia) DeleteExpiredUploads(context.Context, int64) (int, error) { - return 0, storage.ErrNotImplemented -} - -func (notImplementedMedia) Thumbnail(context.Context, string, string, int, int, string) (storage.Thumbnail, error) { - return storage.Thumbnail{}, storage.ErrNotImplemented -} - -func (notImplementedMedia) MediaUsage(context.Context, string, string) (storage.MediaUsage, error) { - return storage.MediaUsage{}, storage.ErrNotImplemented -} - -func (notImplementedMedia) MediaQuota(context.Context, string) (int64, bool, error) { - return 0, false, storage.ErrNotImplemented -} - -func (notImplementedMedia) SetMediaQuota(context.Context, string, int64) error { - return storage.ErrNotImplemented -} - -func (notImplementedMedia) DeleteMediaQuota(context.Context, string) error { - return storage.ErrNotImplemented -} - -func (notImplementedMedia) StoreThumbnail(context.Context, storage.Thumbnail) error { - return storage.ErrNotImplemented +// mediaPrefix is where media content lives in PocketBase's filesystem, +// apart from the files of PocketBase's own collections. +const mediaPrefix = "warp_media" + +// Database is the PocketBase implementation of storage.Database. +type Database struct { + *sqlstore.Store + app *core.BaseApp + blobs *blobStore +} + +// Open bootstraps a PocketBase app in the directory named by +// cfg.ConnectionString — PocketBase's pb_data — and opens the Warp store on +// its database. +func Open(ctx context.Context, cfg storage.Config, log *slog.Logger) (storage.Database, error) { + dir := cfg.ConnectionString + if dir == "" { + return nil, errors.New("pocketbase: connection_string must name the data directory") + } + if err := ctx.Err(); err != nil { + return nil, err + } + + app := core.NewBaseApp(core.BaseAppConfig{ + DataDir: dir, + DataMaxOpenConns: cfg.MaxOpenConns, + DataMaxIdleConns: cfg.MaxIdleConns, + }) + if err := app.Bootstrap(); err != nil { + _ = app.ResetBootstrapState() + return nil, fmt.Errorf("pocketbase: bootstrap %s: %w", dir, err) + } + if cfg.S3.Enabled { + // Applied in memory on every start rather than saved: the Warp + // config is the authority, and PocketBase's own settings would + // otherwise be a second place for the bucket to be set, and to + // disagree. + app.Settings().S3 = core.S3Config{ + Enabled: true, + Bucket: cfg.S3.Bucket, + Region: cfg.S3.Region, + Endpoint: cfg.S3.Endpoint, + AccessKey: cfg.S3.AccessKey, + Secret: cfg.S3.Secret, + ForcePathStyle: cfg.S3.ForcePathStyle, + } + } + + // The writer pool: one connection, the way the sqlite driver runs, and + // the way PocketBase itself writes. Warp's store reads in the same + // transactions it writes in, so it is given the one pool for both. + writer, ok := app.NonconcurrentDB().(*dbx.DB) + if !ok { + _ = app.ResetBootstrapState() + return nil, errors.New("pocketbase: the app's database is not a *dbx.DB") + } + + opts := sqlite.StoreOptions(writer.DB(), log) + opts.DriverName = storage.DriverPocketBase + store, err := sqlstore.New(opts) + if err != nil { + _ = app.ResetBootstrapState() + return nil, err + } + return &Database{ + Store: store, + app: app, + blobs: &blobStore{app: app, tmpDir: filepath.Join(app.DataDir(), ".warp_tmp")}, + }, nil +} + +// App is the PocketBase app, for code that wants PocketBase's own +// features: its collections, its auth, its realtime. +func (d *Database) App() core.App { return d.app } + +// Blobs implements storage.BlobProvider: media content goes into +// PocketBase's filesystem. +func (d *Database) Blobs() storage.BlobStore { return d.blobs } + +// Close implements storage.Database. The app owns the connection pools. +func (d *Database) Close() error { return d.app.ResetBootstrapState() } + +// blobStore is storage.BlobStore over PocketBase's filesystem: the local +// pb_data/storage directory, or the S3 bucket the settings name. +// +// The filesystem can only be written a whole file at a time, and a blob's +// name is its hash, which is not known until the last byte has been read. +// So an upload is spooled to a temporary file first, hashed on the way, and +// handed over once it is complete — which is also what keeps a half-written +// object from ever appearing under a hash. +type blobStore struct { + app *core.BaseApp + tmpDir string +} + +func blobKey(sum string) (string, bool) { + if len(sum) != sha256.Size*2 { + return "", false + } + for _, c := range sum { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return "", false + } + } + return mediaPrefix + "/" + sum[0:2] + "/" + sum[2:4] + "/" + sum, true +} + +// fs opens the filesystem for one operation. It is cheap for both kinds, +// and an instance per operation is what lets each carry its own context. +func (b *blobStore) fs(ctx context.Context) (*filesystem.System, error) { + fsys, err := b.app.NewFilesystem() + if err != nil { + return nil, fmt.Errorf("open the media filesystem: %w", err) + } + fsys.SetContext(ctx) + return fsys, nil +} + +// Put implements storage.BlobStore. +func (b *blobStore) Put(ctx context.Context, r io.Reader, limit int64) (storage.BlobInfo, error) { + if err := os.MkdirAll(b.tmpDir, 0o750); err != nil { + return storage.BlobInfo{}, fmt.Errorf("create the media spool directory: %w", err) + } + tmp, err := os.CreateTemp(b.tmpDir, "upload-*") + if err != nil { + return storage.BlobInfo{}, fmt.Errorf("create a temporary file: %w", err) + } + defer func() { + _ = tmp.Close() + _ = os.Remove(tmp.Name()) + }() + + hash := sha256.New() + n, err := io.Copy(io.MultiWriter(tmp, hash), &ctxReader{ctx: ctx, r: io.LimitReader(r, limit+1)}) + if err != nil { + return storage.BlobInfo{}, fmt.Errorf("store content: %w", err) + } + if n > limit { + return storage.BlobInfo{}, storage.ErrBlobTooLarge + } + if err := tmp.Close(); err != nil { + return storage.BlobInfo{}, fmt.Errorf("store content: %w", err) + } + + info := storage.BlobInfo{SHA256: hex.EncodeToString(hash.Sum(nil)), Size: n} + key, _ := blobKey(info.SHA256) + + fsys, err := b.fs(ctx) + if err != nil { + return storage.BlobInfo{}, err + } + defer func() { _ = fsys.Close() }() + + if exists, err := fsys.Exists(key); err == nil && exists { + // The same content is already stored. + return info, nil + } + file, err := filesystem.NewFileFromPath(tmp.Name()) + if err != nil { + return storage.BlobInfo{}, fmt.Errorf("store content: %w", err) + } + if err := fsys.UploadFile(file, key); err != nil { + return storage.BlobInfo{}, fmt.Errorf("store content: %w", err) + } + return info, nil +} + +// Open implements storage.BlobStore. +func (b *blobStore) Open(ctx context.Context, sum string) (io.ReadSeekCloser, error) { + key, ok := blobKey(sum) + if !ok { + return nil, storage.ErrNoSuchBlob + } + fsys, err := b.fs(ctx) + if err != nil { + return nil, err + } + reader, err := fsys.GetReader(key) + if err != nil { + _ = fsys.Close() + if errors.Is(err, filesystem.ErrNotFound) { + return nil, storage.ErrNoSuchBlob + } + return nil, fmt.Errorf("open media content: %w", err) + } + return &blobReader{ReadSeekCloser: reader, fsys: fsys}, nil +} + +// Delete implements storage.BlobStore. +func (b *blobStore) Delete(ctx context.Context, sum string) error { + key, ok := blobKey(sum) + if !ok { + return nil + } + fsys, err := b.fs(ctx) + if err != nil { + return err + } + defer func() { _ = fsys.Close() }() + if err := fsys.Delete(key); err != nil && !errors.Is(err, filesystem.ErrNotFound) { + return fmt.Errorf("delete media content: %w", err) + } + return nil +} + +// blobReader closes the filesystem with the reader it opened. +type blobReader struct { + io.ReadSeekCloser + fsys *filesystem.System +} + +func (r *blobReader) Close() error { + return errors.Join(r.ReadSeekCloser.Close(), r.fsys.Close()) +} + +// ctxReader stops a copy when its context ends. +type ctxReader struct { + ctx context.Context + r io.Reader +} + +func (c *ctxReader) Read(p []byte) (int, error) { + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.r.Read(p) } -// Compile-time proof that the stub keeps up with the interfaces. var ( - _ storage.Database = (*Database)(nil) - _ storage.UserStore = notImplementedUsers{} - _ storage.RoomStore = notImplementedRooms{} - _ storage.SyncStore = notImplementedSync{} - _ storage.FederationStore = notImplementedFederation{} - _ storage.MediaStore = notImplementedMedia{} - _ storage.KeyStore = notImplementedKeys{} + _ storage.Database = (*Database)(nil) + _ storage.BlobProvider = (*Database)(nil) + _ storage.BlobStore = (*blobStore)(nil) ) - -// notImplementedKeys satisfies storage.KeyStore the same way. A key upload -// that reported success and kept nothing would leave every message sent to -// the device undecryptable. -type notImplementedKeys struct{} - -func (notImplementedKeys) SetDeviceKeys(context.Context, storage.DeviceKeys) (storage.DeviceListUpdate, error) { - return storage.DeviceListUpdate{}, storage.ErrNotImplemented -} - -func (notImplementedKeys) DeviceKeys(context.Context, string, []string) ([]storage.DeviceKeys, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedKeys) DeleteDeviceKeys(context.Context, string, []string) ([]storage.DeviceListUpdate, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedKeys) AddOneTimeKeys(context.Context, string, string, []storage.OneTimeKey) error { - return storage.ErrNotImplemented -} - -func (notImplementedKeys) OneTimeKeyCounts(context.Context, string, string) (map[string]int, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedKeys) SetFallbackKeys(context.Context, string, string, []storage.OneTimeKey) error { - return storage.ErrNotImplemented -} - -func (notImplementedKeys) UnusedFallbackKeyTypes(context.Context, string, string) ([]string, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedKeys) ClaimKey(context.Context, string, string, string) (storage.ClaimedKey, bool, error) { - return storage.ClaimedKey{}, false, storage.ErrNotImplemented -} - -func (notImplementedKeys) SetCrossSigningKeys(context.Context, string, map[string][]byte) (storage.DeviceListUpdate, error) { - return storage.DeviceListUpdate{}, storage.ErrNotImplemented -} - -func (notImplementedKeys) CrossSigningKeys(context.Context, string) (map[string][]byte, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedKeys) AddSignatures(context.Context, []storage.KeySignature) ([]storage.DeviceListUpdate, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedKeys) Signatures(context.Context, string) ([]storage.KeySignature, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedKeys) MarkDeviceListsChanged(context.Context, []string) (int64, error) { - return 0, storage.ErrNotImplemented -} - -func (notImplementedKeys) DeviceListChanges(context.Context, []string, int64, int64) ([]storage.DeviceListChange, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedKeys) DeviceListPosition(context.Context, string) (int64, error) { - return 0, storage.ErrNotImplemented -} - -func (notImplementedKeys) CreateBackup(context.Context, string, string, []byte) (string, error) { - return "", storage.ErrNotImplemented -} - -func (notImplementedKeys) Backup(context.Context, string, string) (storage.BackupVersion, error) { - return storage.BackupVersion{}, storage.ErrNotImplemented -} - -func (notImplementedKeys) UpdateBackup(context.Context, string, string, []byte) error { - return storage.ErrNotImplemented -} - -func (notImplementedKeys) DeleteBackup(context.Context, string, string) error { - return storage.ErrNotImplemented -} - -func (notImplementedKeys) PutBackupKeys(context.Context, string, string, []storage.BackupKey) (storage.BackupVersion, error) { - return storage.BackupVersion{}, storage.ErrNotImplemented -} - -func (notImplementedKeys) BackupKeys(context.Context, string, string, string, string) ([]storage.BackupKey, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedKeys) DeleteBackupKeys(context.Context, string, string, string, string) (storage.BackupVersion, error) { - return storage.BackupVersion{}, storage.ErrNotImplemented -} - -func (notImplementedKeys) RemoteKeys(context.Context, string) (storage.RemoteKeys, bool, error) { - return storage.RemoteKeys{}, false, storage.ErrNotImplemented -} - -func (notImplementedKeys) SetRemoteKeys(context.Context, storage.RemoteKeys) (bool, error) { - return false, storage.ErrNotImplemented -} - -func (notImplementedKeys) UpdateRemoteDevice(context.Context, storage.RemoteDeviceUpdate) (storage.RemoteUpdateResult, error) { - return 0, storage.ErrNotImplemented -} - -func (notImplementedKeys) SetRemoteCrossSigningKeys(context.Context, string, map[string][]byte) (bool, error) { - return false, storage.ErrNotImplemented -} - -func (notImplementedKeys) RemoteKeyUsers(context.Context) ([]string, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedKeys) DeleteRemoteKeys(context.Context, []string) error { - return storage.ErrNotImplemented -} - -// notImplementedPush is the push store until the driver has one. -type notImplementedPush struct{} - -func (notImplementedPush) Pushers(context.Context, string) ([]storage.Pusher, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedPush) SetPusher(context.Context, storage.Pusher, bool) error { - return storage.ErrNotImplemented -} - -func (notImplementedPush) DeletePusher(context.Context, string, string, string) error { - return storage.ErrNotImplemented -} - -func (notImplementedPush) DeletePushersForDevices(context.Context, string, []string) error { - return storage.ErrNotImplemented -} - -func (notImplementedPush) AddNotifications(context.Context, []storage.NewNotification) (int64, error) { - return 0, storage.ErrNotImplemented -} - -func (notImplementedPush) MarkNotificationsRead(context.Context, string, string, int64) (int64, bool, error) { - return 0, false, storage.ErrNotImplemented -} - -func (notImplementedPush) NotificationCounts(context.Context, string, []string) (map[string]storage.NotificationCounts, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedPush) NotificationCountChanges(context.Context, string, int64, int64) ([]string, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedPush) UnreadNotifications(context.Context, string) (int, error) { - return 0, storage.ErrNotImplemented -} - -func (notImplementedPush) Notifications(context.Context, string, int64, int, bool) ([]storage.Notification, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedPush) PruneNotifications(context.Context, int64) (int64, error) { - return 0, storage.ErrNotImplemented -} - -// notImplementedAdmin is the admin store until the driver has one. -type notImplementedAdmin struct{} - -func (notImplementedAdmin) CreateRegistrationToken(context.Context, storage.RegistrationToken) error { - return storage.ErrNotImplemented -} - -func (notImplementedAdmin) RegistrationToken(context.Context, string) (storage.RegistrationToken, error) { - return storage.RegistrationToken{}, storage.ErrNotImplemented -} - -func (notImplementedAdmin) RegistrationTokens(context.Context) ([]storage.RegistrationToken, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedAdmin) UpdateRegistrationToken(context.Context, string, *int, int64) error { - return storage.ErrNotImplemented -} - -func (notImplementedAdmin) DeleteRegistrationToken(context.Context, string) error { - return storage.ErrNotImplemented -} - -func (notImplementedAdmin) UseRegistrationToken(context.Context, string, int64) error { - return storage.ErrNotImplemented -} - -func (notImplementedAdmin) ReleaseRegistrationToken(context.Context, string) error { - return storage.ErrNotImplemented -} - -func (notImplementedAdmin) AppendAudit(context.Context, storage.AuditEntry) (int64, error) { - return 0, storage.ErrNotImplemented -} - -func (notImplementedAdmin) AuditLog(context.Context, storage.AuditFilter) ([]storage.AuditEntry, error) { - return nil, storage.ErrNotImplemented -} - -func (notImplementedAdmin) ServerNoticeRoom(context.Context, string) (string, error) { - return "", storage.ErrNotImplemented -} - -func (notImplementedAdmin) SetServerNoticeRoom(context.Context, string, string) error { - return storage.ErrNotImplemented -} - -// notImplementedAppServices is the application service store until the -// driver has one. -type notImplementedAppServices struct{} - -func (notImplementedAppServices) AppServiceState(context.Context, string, int64) (storage.AppServiceState, error) { - return storage.AppServiceState{}, storage.ErrNotImplemented -} - -func (notImplementedAppServices) BeginTransaction(context.Context, string, int64, int64) error { - return storage.ErrNotImplemented -} - -func (notImplementedAppServices) CompleteTransaction(context.Context, string, int64) error { - return storage.ErrNotImplemented -} diff --git a/server/internal/storage/postgres/migrations/0012_relations_search.sql b/server/internal/storage/postgres/migrations/0012_relations_search.sql new file mode 100644 index 0000000..744bd3c --- /dev/null +++ b/server/internal/storage/postgres/migrations/0012_relations_search.sql @@ -0,0 +1,77 @@ +-- Phase 10: relations (threads, edits, reactions, references) and +-- full-text search. + +-- content.m.relates_to of each event, written out so that "the children of +-- this event" is an index lookup. Both are '' for an event that relates to +-- nothing, and are cleared when the event is redacted: the redaction +-- algorithm strips the content they came from. +ALTER TABLE roomserver_events ADD COLUMN relates_to_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE roomserver_events ADD COLUMN rel_type TEXT NOT NULL DEFAULT ''; + +-- The events already stored. Only a relation with both fields a string +-- counts, which is the rule the roomserver applies to new events. The LIKE +-- keeps the cast to jsonb off the events that cannot have one. +UPDATE roomserver_events e SET + relates_to_id = r.rel ->> 'event_id', + rel_type = r.rel ->> 'rel_type' +FROM ( + SELECT event_id, event_json::jsonb -> 'content' -> 'm.relates_to' AS rel + FROM roomserver_events + WHERE redacted_by = '' AND event_json LIKE '%m.relates_to%' +) r +WHERE e.event_id = r.event_id + AND jsonb_typeof(r.rel -> 'event_id') = 'string' + AND jsonb_typeof(r.rel -> 'rel_type') = 'string'; + +-- /relations, the bundled summaries and /threads all start from a parent +-- and read its children in stream order. +CREATE INDEX IF NOT EXISTS roomserver_events_relations_idx + ON roomserver_events (room_id, relates_to_id, rel_type, stream_ordering); + +-- The full-text index /search reads: one row per searchable event, keyed by +-- the event's stream ordering, which is unique across the server and is +-- what a redaction finds the row by. Outliers are not here: an event joins +-- the index when it joins a timeline. +-- +-- The 'simple' configuration lower-cases and splits on word boundaries +-- without stemming for any one language, which is the right default for a +-- server whose rooms are in every language at once. +CREATE TABLE IF NOT EXISTS roomserver_event_search ( + stream_ordering BIGINT NOT NULL PRIMARY KEY, + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + search_key TEXT NOT NULL, + sender TEXT NOT NULL, + event_type TEXT NOT NULL, + body TEXT NOT NULL, + vector TSVECTOR GENERATED ALWAYS AS (to_tsvector('simple', body)) STORED +); + +CREATE INDEX IF NOT EXISTS roomserver_event_search_vector_idx + ON roomserver_event_search USING GIN (vector); +CREATE INDEX IF NOT EXISTS roomserver_event_search_room_idx + ON roomserver_event_search (room_id, stream_ordering); + +INSERT INTO roomserver_event_search (stream_ordering, room_id, event_id, search_key, sender, event_type, body) +SELECT stream_ordering, room_id, event_id, k.search_key, sender, event_type, k.body +FROM ( + SELECT stream_ordering, room_id, event_id, sender, event_type, event_json::jsonb -> 'content' AS content + FROM roomserver_events + WHERE event_type IN ('m.room.message', 'm.room.name', 'm.room.topic') + AND NOT outlier AND redacted_by = '' +) e +CROSS JOIN LATERAL ( + SELECT CASE e.event_type + WHEN 'm.room.message' THEN 'content.body' + WHEN 'm.room.name' THEN 'content.name' + ELSE 'content.topic' + END AS search_key, + CASE e.event_type + WHEN 'm.room.message' THEN e.content ->> 'body' + WHEN 'm.room.name' THEN e.content ->> 'name' + ELSE e.content ->> 'topic' + END AS body +) k +WHERE k.body IS NOT NULL + AND jsonb_typeof(e.content -> substr(k.search_key, 9)) = 'string' +ON CONFLICT (stream_ordering) DO NOTHING; diff --git a/server/internal/storage/postgres/migrations/0013_thread_notifications.sql b/server/internal/storage/postgres/migrations/0013_thread_notifications.sql new file mode 100644 index 0000000..f86ee84 --- /dev/null +++ b/server/internal/storage/postgres/migrations/0013_thread_notifications.sql @@ -0,0 +1,21 @@ +-- Phase 10: notifications counted per thread. +-- +-- thread_id is the thread an event that notified a user belongs to: the +-- root its m.relates_to names when rel_type is m.thread, and '' for an +-- event on the main timeline. A read receipt for a thread clears that +-- thread's rows and a receipt for the main timeline clears the '' rows, +-- which is what lets reading one thread leave the rest of the room unread. +ALTER TABLE push_notifications ADD COLUMN thread_id TEXT NOT NULL DEFAULT ''; + +-- The notifications already stored, from the relation 0012 projected onto +-- their events. A notification whose event has gone stays on the main +-- timeline, where an unthreaded or main receipt still clears it. +UPDATE push_notifications SET thread_id = COALESCE(( + SELECT e.relates_to_id FROM roomserver_events e + WHERE e.event_id = push_notifications.event_id AND e.rel_type = 'm.thread' +), ''); + +-- A threaded receipt marks one thread's rows read; the per-thread counts a +-- sync asks for group a room's unread rows by thread. +CREATE INDEX IF NOT EXISTS push_notifications_thread_idx + ON push_notifications (user_id, room_id, thread_id, is_read, event_pos); diff --git a/server/internal/storage/postgres/postgres.go b/server/internal/storage/postgres/postgres.go index 6bd0710..7eac72b 100644 --- a/server/internal/storage/postgres/postgres.go +++ b/server/internal/storage/postgres/postgres.go @@ -9,6 +9,7 @@ import ( "embed" "fmt" "log/slog" + "strings" _ "github.com/jackc/pgx/v5/stdlib" // registers the "pgx" database/sql driver @@ -44,6 +45,33 @@ var settingsQueries = sqlstore.SettingsQueries{ Delete: `DELETE FROM server_settings WHERE setting_key = $1`, } +// searchDialect is the full-text index on tsvector, with the 'simple' +// configuration the migration builds the column with. +var searchDialect = &sqlstore.SearchDialect{ + Insert: `INSERT INTO roomserver_event_search (stream_ordering, room_id, event_id, search_key, sender, event_type, body) + VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT (stream_ordering) DO NOTHING`, + DeleteEvent: `DELETE FROM roomserver_event_search WHERE room_id = ? AND event_id = ?`, + DeleteRoom: `DELETE FROM roomserver_event_search WHERE room_id = ?`, + From: `roomserver_event_search s`, + Name: `s`, + Position: `s.stream_ordering`, + Match: `s.vector @@ to_tsquery('simple', ?)`, + Rank: `ts_rank(s.vector, to_tsquery('simple', ?))`, + RankArgs: 1, + Query: tsQuery, +} + +// tsQuery builds a tsquery from a search term: every word as a prefix +// match, ANDed. The words are letters and digits only, so none of them can +// carry tsquery syntax of its own. +func tsQuery(term string) string { + words := storage.SearchWords(term) + for i, w := range words { + words[i] = w + ":*" + } + return strings.Join(words, " & ") +} + // Database is the PostgreSQL implementation of storage.Database. type Database struct { *sqlstore.Store @@ -77,6 +105,7 @@ func Open(ctx context.Context, cfg storage.Config, log *slog.Logger) (storage.Da Settings: settingsQueries, Rebind: rebind, IsUniqueViolation: isUniqueViolation, + Search: searchDialect, Logger: log, }) if err != nil { diff --git a/server/internal/storage/push.go b/server/internal/storage/push.go index e478b7b..a68989b 100644 --- a/server/internal/storage/push.go +++ b/server/internal/storage/push.go @@ -47,6 +47,11 @@ type NewNotification struct { // what a read receipt is compared against. EventPos int64 + // ThreadID is the thread the event is in: the root its m.relates_to + // names when rel_type is m.thread, or ThreadMain for an event on the + // main timeline. A thread root is itself on the main timeline. + ThreadID string + TimestampMS int64 Highlight bool @@ -66,6 +71,19 @@ type Notification struct { Read bool } +// Threads a read receipt can be for, as the thread argument of +// MarkNotificationsRead. Any other value is a thread root's event ID. +const ( + // ThreadMain is the main timeline: the thread ID of an event in no + // thread, and what a receipt with thread_id "main" reads. + ThreadMain = "" + + // AllThreads is an unthreaded receipt, which reads the main timeline + // and every thread together. It cannot be mistaken for a thread root: + // an event ID starts with '$'. + AllThreads = "*" +) + // NotificationCounts is a room's unread notifications for one user. type NotificationCounts struct { Notifications int @@ -105,13 +123,22 @@ type PushStore interface { // MarkNotificationsRead marks a user's notifications in a room up to // and including an event position as read, and reports whether the - // counts moved. - MarkNotificationsRead(ctx context.Context, userID, roomID string, upTo int64) (pos int64, moved bool, err error) + // counts moved. thread limits it to one thread — ThreadMain for the + // main timeline, a root's event ID for a thread — or is AllThreads. + MarkNotificationsRead(ctx context.Context, userID, roomID, thread string, upTo int64) (pos int64, moved bool, err error) // NotificationCounts reads a user's unread counts in the given rooms. // A room with nothing unread is in the result with zero counts. NotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]NotificationCounts, error) + // ThreadNotificationCounts reads a user's unread counts in the given + // rooms one thread at a time, keyed by room and then by thread ID, with + // the main timeline under ThreadMain. Only threads with something + // unread are in the result. The room's move in the notification stream + // covers its threads: any change to a thread's counts is a change to + // the room's. + ThreadNotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]map[string]NotificationCounts, error) + // NotificationCountChanges lists the rooms whose counts for a user // moved after from and at or before to. NotificationCountChanges(ctx context.Context, userID string, from, to int64) ([]string, error) diff --git a/server/internal/storage/push_test.go b/server/internal/storage/push_test.go index 2d247f9..0c67260 100644 --- a/server/internal/storage/push_test.go +++ b/server/internal/storage/push_test.go @@ -2,7 +2,6 @@ package storage_test import ( "database/sql" - "errors" "os" "slices" "testing" @@ -151,7 +150,7 @@ func TestNotifications(t *testing.T) { // Reading up to the first event clears it alone, and moves the // stream for that room only. - pos, moved, err := push.MarkNotificationsRead(ctx, alice, room1, 10) + pos, moved, err := push.MarkNotificationsRead(ctx, alice, room1, storage.AllThreads, 10) if err != nil || !moved || pos != last+1 { t.Fatalf("MarkNotificationsRead = %d, %v, %v", pos, moved, err) } @@ -164,11 +163,11 @@ func TestNotifications(t *testing.T) { } // Reading the same again moves nothing. - if again, moved, err := push.MarkNotificationsRead(ctx, alice, room1, 10); err != nil || moved || again != pos { + if again, moved, err := push.MarkNotificationsRead(ctx, alice, room1, storage.AllThreads, 10); err != nil || moved || again != pos { t.Errorf("reading again = %d, %v, %v", again, moved, err) } - if _, _, err := push.MarkNotificationsRead(ctx, alice, room1, 100); err != nil { + if _, _, err := push.MarkNotificationsRead(ctx, alice, room1, storage.AllThreads, 100); err != nil { t.Fatal(err) } counts, _ = push.NotificationCounts(ctx, alice, []string{room1}) @@ -206,13 +205,101 @@ func TestNotifications(t *testing.T) { } } -func TestPushNotImplementedOnPocketBase(t *testing.T) { - db, err := storage.Open(t.Context(), storage.Config{Driver: storage.DriverPocketBase}, nil) - if err != nil { - t.Skipf("pocketbase: %v", err) - } - defer func() { _ = db.Close() }() - if _, err := db.Push().Pushers(t.Context(), "alice"); !errors.Is(err, storage.ErrNotImplemented) { - t.Errorf("Pushers = %v", err) +// TestThreadNotifications covers notifications filed by thread: a receipt +// for one thread, for the main timeline, and for the whole room each read +// only their own part, while the room totals stay the sum of every part. +func TestThreadNotifications(t *testing.T) { + const alice = "@alice:warp.example" + const room, other = "!one:warp.example", "!two:warp.example" + const t1, t2 = "$root1", "$root2" + + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + push := migratedPushDB(t, driver).Push() + + note := func(event, thread string, pos int64, highlight bool) storage.NewNotification { + return storage.NewNotification{UserID: alice, RoomID: room, EventID: event, EventPos: pos, + ThreadID: thread, TimestampMS: 1000 + pos, Highlight: highlight, Actions: []byte(`["notify"]`)} + } + if _, err := push.AddNotifications(ctx, []storage.NewNotification{ + note(t1, storage.ThreadMain, 10, false), // a thread root is on the main timeline + note("$a", t1, 11, true), + note("$b", t1, 12, false), + note("$c", t2, 13, false), + note("$d", storage.ThreadMain, 14, false), + }); err != nil { + t.Fatal(err) + } + + byThread := func() map[string]storage.NotificationCounts { + t.Helper() + counts, err := push.ThreadNotificationCounts(ctx, alice, []string{room, other}) + if err != nil { + t.Fatal(err) + } + if _, ok := counts[other]; ok { + t.Errorf("a room with nothing unread is in the result: %+v", counts[other]) + } + return counts[room] + } + total := func() storage.NotificationCounts { + t.Helper() + counts, err := push.NotificationCounts(ctx, alice, []string{room}) + if err != nil { + t.Fatal(err) + } + return counts[room] + } + + got := byThread() + if len(got) != 3 || + got[storage.ThreadMain] != (storage.NotificationCounts{Notifications: 2}) || + got[t1] != (storage.NotificationCounts{Notifications: 2, Highlights: 1}) || + got[t2] != (storage.NotificationCounts{Notifications: 1}) { + t.Errorf("ThreadNotificationCounts = %+v", got) + } + if all, _ := push.Notifications(ctx, alice, 0, 10, false); len(all) != 5 || all[1].ThreadID != t2 { + t.Errorf("Notifications = %+v", all) + } + + // A receipt in thread 1 reads that thread, past everything else + // in the room, and leaves the rest unread. + if _, moved, err := push.MarkNotificationsRead(ctx, alice, room, t1, 100); err != nil || !moved { + t.Fatalf("reading thread 1 = %v, %v", moved, err) + } + got = byThread() + if _, ok := got[t1]; ok || len(got) != 2 { + t.Errorf("after reading thread 1: %+v", got) + } + if c := total(); c != (storage.NotificationCounts{Notifications: 3}) { + t.Errorf("the room total after reading thread 1 = %+v", c) + } + + // A main receipt reads the main timeline up to its event. + if _, moved, err := push.MarkNotificationsRead(ctx, alice, room, storage.ThreadMain, 10); err != nil || !moved { + t.Fatalf("reading the main timeline = %v, %v", moved, err) + } + got = byThread() + if got[storage.ThreadMain] != (storage.NotificationCounts{Notifications: 1}) || + got[t2] != (storage.NotificationCounts{Notifications: 1}) { + t.Errorf("after the main receipt: %+v", got) + } + // A receipt for a thread with nothing unread moves nothing. + if _, moved, err := push.MarkNotificationsRead(ctx, alice, room, t1, 100); err != nil || moved { + t.Errorf("reading thread 1 again = %v, %v", moved, err) + } + + // An unthreaded receipt reads everything. + if _, _, err := push.MarkNotificationsRead(ctx, alice, room, storage.AllThreads, 100); err != nil { + t.Fatal(err) + } + if got := byThread(); len(got) != 0 { + t.Errorf("after the unthreaded receipt: %+v", got) + } + if c := total(); c != (storage.NotificationCounts{}) { + t.Errorf("the room total after the unthreaded receipt = %+v", c) + } + }) } } diff --git a/server/internal/storage/relations.go b/server/internal/storage/relations.go new file mode 100644 index 0000000..5acf592 --- /dev/null +++ b/server/internal/storage/relations.go @@ -0,0 +1,189 @@ +package storage + +import "context" + +// The relation and search reads of the room store: threads, edits, +// reactions and references (an event's content.m.relates_to), and the +// full-text index /search is answered from. +// +// Both are projections of the event JSON that the roomserver works out and +// hands over with the event, the same as its type and state key: the store +// still never parses an event. They live in a file of their own because +// they are the Phase 10 additions to the room store, and because neither is +// about a room's state or its stream order — they are about which events +// point at which, and which events say what. + +// Relation types the server aggregates. Every rel_type is recorded and can +// be listed through /relations; these are the ones whose summaries go into +// an event's unsigned.m.relations. +const ( + RelThread = "m.thread" + RelReplace = "m.replace" + RelReference = "m.reference" + RelAnnotate = "m.annotation" +) + +// RelationsRequest is one page of the events relating to some parents. +// +// The bounds are a TimelineRequest's: Low exclusive, High inclusive, zero +// unbounded at either end, positions possibly negative inside backfilled +// history. +type RelationsRequest struct { + RoomID string + + // ParentIDs are the events whose children are read. It is one event + // for a plain /relations call and the event plus its descendants for a + // recursive one. + ParentIDs []string + + // RelType and EventType narrow the children, when set. + RelType string + EventType string + + Low int64 + High int64 + Direction TimelineDirection + + // Limit caps the rows returned, and is required. + Limit int +} + +// RelationAggregate is what one parent's children of one relation type +// come to, from one sender. +// +// It is per sender because two of the summaries need it: an edit only +// counts when it comes from the original's sender, and a thread reports +// whether the user asking took part. Summing over senders is then the +// roomserver's job. +type RelationAggregate struct { + ParentID string + RelType string + Sender string + + // Count is how many children there are. + Count int + + // LatestEventID is the newest of them, by stream order. + LatestEventID string + + // LatestOrdering is its position. + LatestOrdering int64 +} + +// ThreadsRequest is one page of a room's threads, newest activity first. +type ThreadsRequest struct { + RoomID string + + // ParticipatedBy, when set, keeps only the threads that user started + // or replied in. + ParticipatedBy string + + // Before is an exclusive bound on a thread's latest activity: the page + // after one whose last thread was last active at Before. Zero is the + // first page. + Before int64 + + // High bounds what counts as activity, inclusive, for a user whose + // view of the room ends at their leave. Zero is unbounded. + High int64 + + // Limit caps the threads returned, and is required. + Limit int +} + +// Thread is one thread of a room. +type Thread struct { + RootID string + + // LatestOrdering is the position of the newest event in the thread, + // which is what the listing is ordered and paged by. + LatestOrdering int64 +} + +// SearchOrder is how /search orders its results. +type SearchOrder string + +// The two orders the Client-Server API defines. +const ( + SearchByRank SearchOrder = "rank" + SearchByRecent SearchOrder = "recent" +) + +// SearchRange is the part of one room a user may search: the window of its +// stream their membership and the room's history visibility allow. The +// bounds are a TimelineRequest's. +type SearchRange struct { + RoomID string + Low int64 + High int64 +} + +// SearchRequest is one page of a full-text search. +type SearchRequest struct { + // Term is what the user typed. The store turns it into its engine's + // query syntax; a term with no searchable words matches nothing. + Term string + + // Rooms are where to look, each bounded to what the user may see. + Rooms []SearchRange + + // Keys narrows which fields match: "content.body", "content.name", + // "content.topic". Empty means all three. + Keys []string + + // Filter narrows by type and sender, as a timeline filter does. + Filter TimelineFilter + + Order SearchOrder + + // Offset skips results, for paging a ranked search. A search by + // recency pages by Before instead, which stays put as events arrive. + Offset int + + // Before is an exclusive upper bound on the stream position, for + // paging a search by recency. Zero is the first page. + Before int64 + + // Limit caps the results returned, and is required. + Limit int +} + +// SearchResult is one event a search found. +type SearchResult struct { + Event StoredEvent + + // Rank is the engine's relevance score, higher is better. It is only + // comparable within one search. + Rank float64 +} + +// RelationStore is the part of the room store about relations and search. +// It is embedded in RoomStore; it is a separate interface only so that the +// two halves can be read apart. +type RelationStore interface { + // Relations reads one page of the children of the given parents, in + // stream order in the direction asked for. + Relations(ctx context.Context, req RelationsRequest) ([]StoredEvent, error) + + // ChildEventIDs lists the direct children of the given parents, + // narrowed by relation and event type when those are set. It is how a + // recursive /relations walks down a level at a time. + ChildEventIDs(ctx context.Context, roomID string, parentIDs []string, relType, eventType string) ([]string, error) + + // RelationAggregates summarises the thread, edit and reference + // children of the given parents, up to and including high (zero for + // no bound). + RelationAggregates(ctx context.Context, roomID string, parentIDs []string, high int64) ([]RelationAggregate, error) + + // RelationEventIDs lists the children of one relation type of the + // given parents, oldest first, at most limit of them: the chunk of an + // m.reference summary. + RelationEventIDs(ctx context.Context, roomID string, parentIDs []string, relType string, high int64, limit int) (map[string][]string, error) + + // Threads reads one page of a room's threads. + Threads(ctx context.Context, req ThreadsRequest) ([]Thread, error) + + // Search reads one page of a full-text search and the total number of + // matches. + Search(ctx context.Context, req SearchRequest) ([]SearchResult, int, error) +} diff --git a/server/internal/storage/relations_test.go b/server/internal/storage/relations_test.go new file mode 100644 index 0000000..6547ca1 --- /dev/null +++ b/server/internal/storage/relations_test.go @@ -0,0 +1,279 @@ +package storage_test + +import ( + "slices" + "testing" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// writeMessage appends a timeline event with the given projections and +// returns its stream ordering. +func writeMessage(t *testing.T, rooms storage.RoomStore, ev storage.StoredEvent) int64 { + t.Helper() + pos, err := rooms.Write(t.Context(), storage.RoomWrite{Event: ev, Extremities: []string{ev.EventID}}) + if err != nil { + t.Fatalf("Write %s: %v", ev.EventID, err) + } + return pos +} + +func relationEvent(id, sender, parent, relType string) storage.StoredEvent { + e := newTestEvent(testRoomID, id, "m.room.message", nil) + e.Sender = sender + e.RelatesTo, e.RelType = parent, relType + return e +} + +func TestRelationsAndAggregates(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + rooms := migratedRoomDB(t, driver).Rooms() + if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { + t.Fatalf("CreateRoom: %v", err) + } + + writeMessage(t, rooms, newTestEvent(testRoomID, "$root", "m.room.message", nil)) + writeMessage(t, rooms, relationEvent("$t1", "@bob:warp.example", "$root", storage.RelThread)) + last := writeMessage(t, rooms, relationEvent("$t2", "@carol:warp.example", "$root", storage.RelThread)) + writeMessage(t, rooms, relationEvent("$edit", "@alice:warp.example", "$root", storage.RelReplace)) + writeMessage(t, rooms, relationEvent("$ref", "@bob:warp.example", "$root", storage.RelReference)) + writeMessage(t, rooms, relationEvent("$t1edit", "@bob:warp.example", "$t1", storage.RelReplace)) + + children, err := rooms.Relations(ctx, storage.RelationsRequest{ + RoomID: testRoomID, ParentIDs: []string{"$root"}, + Direction: storage.DirectionBackward, Limit: 10, + }) + if err != nil { + t.Fatalf("Relations: %v", err) + } + if got := eventIDs(children); !slices.Equal(got, []string{"$ref", "$edit", "$t2", "$t1"}) { + t.Errorf("Relations = %v", got) + } + if children[0].RelatesTo != "$root" || children[0].RelType != storage.RelReference { + t.Errorf("the projections did not round-trip: %+v", children[0]) + } + + threads, err := rooms.Relations(ctx, storage.RelationsRequest{ + RoomID: testRoomID, ParentIDs: []string{"$root", "$t1"}, RelType: storage.RelReplace, + Direction: storage.DirectionForward, Limit: 10, + }) + if err != nil { + t.Fatalf("Relations by type: %v", err) + } + if got := eventIDs(threads); !slices.Equal(got, []string{"$edit", "$t1edit"}) { + t.Errorf("m.replace children of two parents = %v", got) + } + + ids, err := rooms.ChildEventIDs(ctx, testRoomID, []string{"$t1"}, "", "") + if err != nil || !slices.Equal(ids, []string{"$t1edit"}) { + t.Errorf("ChildEventIDs = %v, %v", ids, err) + } + + aggregates, err := rooms.RelationAggregates(ctx, testRoomID, []string{"$root"}, 0) + if err != nil { + t.Fatalf("RelationAggregates: %v", err) + } + threadCount, latest := 0, "" + for _, a := range aggregates { + if a.RelType == storage.RelThread { + threadCount += a.Count + if a.LatestOrdering == last { + latest = a.LatestEventID + } + } + } + if threadCount != 2 || latest != "$t2" || len(aggregates) != 4 { + t.Errorf("aggregates = %+v", aggregates) + } + + refs, err := rooms.RelationEventIDs(ctx, testRoomID, []string{"$root"}, storage.RelReference, 0, 10) + if err != nil || !slices.Equal(refs["$root"], []string{"$ref"}) { + t.Errorf("RelationEventIDs = %v, %v", refs, err) + } + + // Bounded above, the later replies are not counted. + bounded, err := rooms.RelationAggregates(ctx, testRoomID, []string{"$root"}, last-1) + if err != nil { + t.Fatalf("RelationAggregates bounded: %v", err) + } + for _, a := range bounded { + if a.RelType == storage.RelThread && a.LatestEventID == "$t2" { + t.Errorf("a bounded aggregate saw the event above its bound: %+v", a) + } + } + }) + } +} + +func TestThreadsListing(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + rooms := migratedRoomDB(t, driver).Rooms() + if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { + t.Fatalf("CreateRoom: %v", err) + } + writeMessage(t, rooms, newTestEvent(testRoomID, "$a", "m.room.message", nil)) + b := newTestEvent(testRoomID, "$b", "m.room.message", nil) + b.Sender = "@bob:warp.example" + writeMessage(t, rooms, b) + writeMessage(t, rooms, relationEvent("$a1", "@alice:warp.example", "$a", storage.RelThread)) + writeMessage(t, rooms, relationEvent("$b1", "@alice:warp.example", "$b", storage.RelThread)) + // A thread whose root this server does not hold is not listed. + writeMessage(t, rooms, relationEvent("$x1", "@alice:warp.example", "$missing", storage.RelThread)) + + all, err := rooms.Threads(ctx, storage.ThreadsRequest{RoomID: testRoomID, Limit: 10}) + if err != nil { + t.Fatalf("Threads: %v", err) + } + if got := threadRoots(all); !slices.Equal(got, []string{"$b", "$a"}) { + t.Errorf("Threads = %v, want newest activity first", got) + } + + page, err := rooms.Threads(ctx, storage.ThreadsRequest{RoomID: testRoomID, Limit: 1, Before: all[0].LatestOrdering}) + if err != nil || !slices.Equal(threadRoots(page), []string{"$a"}) { + t.Errorf("Threads after the first = %v, %v", threadRoots(page), err) + } + + bobs, err := rooms.Threads(ctx, storage.ThreadsRequest{ + RoomID: testRoomID, ParticipatedBy: "@bob:warp.example", Limit: 10, + }) + if err != nil || !slices.Equal(threadRoots(bobs), []string{"$b"}) { + t.Errorf("Threads bob took part in = %v, %v", threadRoots(bobs), err) + } + }) + } +} + +func TestSearch(t *testing.T) { + for _, driver := range testDrivers { + t.Run(driver, func(t *testing.T) { + ctx := t.Context() + rooms := migratedRoomDB(t, driver).Rooms() + const other = "!other:warp.example" + for _, id := range []string{testRoomID, other} { + if err := rooms.CreateRoom(ctx, newTestRoom(id)); err != nil { + t.Fatalf("CreateRoom: %v", err) + } + } + + message := func(roomID, id, body string) int64 { + e := newTestEvent(roomID, id, "m.room.message", nil) + e.SearchKey, e.SearchText = "content.body", body + return writeMessage(t, rooms, e) + } + message(testRoomID, "$hello", "Hello wonderful world") + message(testRoomID, "$cafe", "Meet at the café tomorrow") + second := message(testRoomID, "$hello2", "hello again, hello") + message(other, "$elsewhere", "hello from elsewhere") + topic := newTestEvent(testRoomID, "$topic", "m.room.topic", stateKey("")) + topic.SearchKey, topic.SearchText = "content.topic", "a room to say hello in" + writeMessage(t, rooms, topic) + + search := func(req storage.SearchRequest) ([]string, int) { + t.Helper() + if req.Limit == 0 { + req.Limit = 10 + } + if req.Rooms == nil { + req.Rooms = []storage.SearchRange{{RoomID: testRoomID}} + } + results, total, err := rooms.Search(ctx, req) + if err != nil { + t.Fatalf("Search(%q): %v", req.Term, err) + } + out := make([]string, 0, len(results)) + for _, r := range results { + out = append(out, r.Event.EventID) + } + return out, total + } + + got, total := search(storage.SearchRequest{Term: "hello", Order: storage.SearchByRecent}) + if !slices.Equal(got, []string{"$topic", "$hello2", "$hello"}) || total != 3 { + t.Errorf("recent search for hello = %v (total %d)", got, total) + } + // A prefix matches, and diacritics are folded on SQLite; the + // tsvector 'simple' configuration keeps them, so the term is + // spelled as stored. + if got, _ := search(storage.SearchRequest{Term: "hel"}); len(got) != 3 { + t.Errorf("prefix search = %v", got) + } + if got, _ := search(storage.SearchRequest{Term: "café"}); !slices.Equal(got, []string{"$cafe"}) { + t.Errorf("search for café = %v", got) + } + // Operators in the term are words, not syntax. + if got, _ := search(storage.SearchRequest{Term: `"hello" & (world`}); !slices.Equal(got, []string{"$hello"}) { + t.Errorf("a term full of operators = %v, want the one event with both words", got) + } + if got, _ := search(storage.SearchRequest{Term: "!!!"}); len(got) != 0 { + t.Errorf("a term with no words found %v", got) + } + + if got, _ := search(storage.SearchRequest{Term: "hello", Keys: []string{"content.topic"}}); !slices.Equal(got, []string{"$topic"}) { + t.Errorf("search by key = %v", got) + } + if got, _ := search(storage.SearchRequest{ + Term: "hello", Rooms: []storage.SearchRange{{RoomID: testRoomID, High: second - 1}}, + }); !slices.Equal(got, []string{"$hello"}) { + t.Errorf("search bounded below the second hello = %v", got) + } + if got, total := search(storage.SearchRequest{ + Term: "hello", Rooms: []storage.SearchRange{{RoomID: testRoomID}, {RoomID: other}}, + }); len(got) != 4 || total != 4 { + t.Errorf("search across two rooms = %v (total %d)", got, total) + } + if got, _ := search(storage.SearchRequest{ + Term: "hello", Filter: storage.TimelineFilter{NotTypes: []string{"m.room.topic"}}, Order: storage.SearchByRecent, + }); !slices.Equal(got, []string{"$hello2", "$hello"}) { + t.Errorf("filtered search = %v", got) + } + + // Paging by recency, then by rank. + page1, _ := search(storage.SearchRequest{Term: "hello", Order: storage.SearchByRecent, Limit: 2}) + if len(page1) != 2 { + t.Fatalf("first page = %v", page1) + } + if page2, _ := search(storage.SearchRequest{Term: "hello", Order: storage.SearchByRank, Offset: 2, Limit: 2}); len(page2) != 1 { + t.Errorf("ranked second page = %v", page2) + } + + // A redacted event is no longer found by what it said. + redaction := newTestEvent(testRoomID, "$redaction", "m.room.redaction", nil) + if _, err := rooms.Write(ctx, storage.RoomWrite{ + Event: redaction, Extremities: []string{"$redaction"}, + Redacts: "$hello2", RedactedJSON: []byte(`{"type":"m.room.message"}`), + }); err != nil { + t.Fatalf("Write redaction: %v", err) + } + if got, _ := search(storage.SearchRequest{Term: "again"}); len(got) != 0 { + t.Errorf("the redacted event is still found: %v", got) + } + + if err := rooms.PurgeRoom(ctx, testRoomID); err != nil { + t.Fatalf("PurgeRoom: %v", err) + } + if got, _ := search(storage.SearchRequest{Term: "hello"}); len(got) != 0 { + t.Errorf("a purged room is still searched: %v", got) + } + }) + } +} + +func eventIDs(events []storage.StoredEvent) []string { + out := make([]string, 0, len(events)) + for _, e := range events { + out = append(out, e.EventID) + } + return out +} + +func threadRoots(threads []storage.Thread) []string { + out := make([]string, 0, len(threads)) + for _, t := range threads { + out = append(out, t.RootID) + } + return out +} diff --git a/server/internal/storage/rooms.go b/server/internal/storage/rooms.go index 6d35efc..82065a7 100644 --- a/server/internal/storage/rooms.go +++ b/server/internal/storage/rooms.go @@ -140,6 +140,22 @@ type StoredEvent struct { // soft-failed. An outlier can be cited and read by ID, and is never in // a timeline. Outlier bool + + // RelatesTo and RelType are content.m.relates_to's event_id and + // rel_type, empty for an event that relates to nothing. A reply's + // m.in_reply_to is not a relation in this sense and is not recorded. + // They are cleared when the event is redacted, because the redaction + // algorithm strips the content they came from. + RelatesTo string + RelType string + + // SearchKey and SearchText are what the full-text index holds for the + // event: the field ("content.body", "content.name" or "content.topic") + // and its text. Both are empty for an event with nothing to search. + // They are written, never read back: the index is the only place they + // are kept. + SearchKey string + SearchText string } // IsState reports whether the event is a state event. @@ -302,6 +318,8 @@ type RoomStats struct { // guarantees is that a room's events, its state and its extremities are only // ever seen in a consistent set. type RoomStore interface { + RelationStore + // CreateRoom inserts the room row, returning ErrRoomExists when the ID // is taken. CreateRoom(ctx context.Context, room NewRoom) error @@ -398,6 +416,13 @@ type RoomStore interface { // otherwise need two hundred timeline reads to find that out. RoomsWithEvents(ctx context.Context, roomIDs []string, from, to int64) ([]string, error) + // LatestEventPositions returns, for each of roomIDs that has one, the + // position of its newest timeline event of one of the given types at + // or before upTo (zero for no bound). It is what sliding sync orders a + // room list by: the last thing that happened in each room that a user + // would call activity. + LatestEventPositions(ctx context.Context, roomIDs, types []string, upTo int64) (map[string]int64, error) + // StateAtStream returns the room's state as of a stream position: the // state after the last event at or before pos. The position may be a // negative one inside backfilled history; zero has no events at or diff --git a/server/internal/storage/rooms_test.go b/server/internal/storage/rooms_test.go index 9240989..9c9210e 100644 --- a/server/internal/storage/rooms_test.go +++ b/server/internal/storage/rooms_test.go @@ -42,7 +42,7 @@ func resetRoomTables(t *testing.T, driver string) { roomserver_state_entries, roomserver_state_snapshots, roomserver_current_state, roomserver_extremities, roomserver_transactions, roomserver_aliases, roomserver_invite_state, - roomserver_events, roomserver_rooms`); err != nil { + roomserver_events, roomserver_rooms, roomserver_event_search`); err != nil { t.Fatalf("reset the room tables: %v", err) } if _, err := db.ExecContext(t.Context(), diff --git a/server/internal/storage/searchwords.go b/server/internal/storage/searchwords.go new file mode 100644 index 0000000..649a5b5 --- /dev/null +++ b/server/internal/storage/searchwords.go @@ -0,0 +1,26 @@ +package storage + +import ( + "strings" + "unicode" +) + +// SearchWords splits what a user typed into a search box into the words +// an index can match: runs of letters and digits, lower-cased, everything +// else a separator. +// +// Both engines' query languages have operators — FTS5's quotes, AND, NEAR +// and column filters, tsquery's & | ! and parentheses — and a search box is +// not the place to expose them: a user searching for "C++ & Go" means the +// words, not a malformed query. Reducing the term to words and building the +// query from those is what makes any input safe to pass on. +func SearchWords(term string) []string { + fields := strings.FieldsFunc(term, func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) + out := fields[:0] + for _, f := range fields { + out = append(out, strings.ToLower(f)) + } + return out +} diff --git a/server/internal/storage/sqlite/migrations/0012_relations_search.sql b/server/internal/storage/sqlite/migrations/0012_relations_search.sql new file mode 100644 index 0000000..5177620 --- /dev/null +++ b/server/internal/storage/sqlite/migrations/0012_relations_search.sql @@ -0,0 +1,61 @@ +-- Phase 10: relations (threads, edits, reactions, references) and +-- full-text search. + +-- content.m.relates_to of each event, written out so that "the children of +-- this event" is an index lookup. Both are '' for an event that relates to +-- nothing, and are cleared when the event is redacted: the redaction +-- algorithm strips the content they came from. +ALTER TABLE roomserver_events ADD COLUMN relates_to_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE roomserver_events ADD COLUMN rel_type TEXT NOT NULL DEFAULT ''; + +-- The events already stored. Only a relation with both fields a string +-- counts, which is the rule the roomserver applies to new events. +UPDATE roomserver_events SET + relates_to_id = json_extract(event_json, '$.content."m.relates_to".event_id'), + rel_type = json_extract(event_json, '$.content."m.relates_to".rel_type') +WHERE redacted_by = '' + AND json_type(event_json, '$.content."m.relates_to".event_id') = 'text' + AND json_type(event_json, '$.content."m.relates_to".rel_type') = 'text'; + +-- /relations, the bundled summaries and /threads all start from a parent +-- and read its children in stream order. +CREATE INDEX IF NOT EXISTS roomserver_events_relations_idx + ON roomserver_events (room_id, relates_to_id, rel_type, stream_ordering); + +-- The full-text index /search reads: one row per searchable event, keyed by +-- the event's stream ordering as its rowid, which is unique across the +-- server and is what a redaction finds the row by. Outliers are not here: +-- an event joins the index when it joins a timeline. +-- +-- unicode61 with diacritics removed matches "cafe" against "café", which is +-- what a user searching a chat history expects. +CREATE VIRTUAL TABLE IF NOT EXISTS roomserver_event_search USING fts5( + body, + room_id UNINDEXED, + event_id UNINDEXED, + search_key UNINDEXED, + sender UNINDEXED, + event_type UNINDEXED, + tokenize = 'unicode61 remove_diacritics 2' +); + +INSERT INTO roomserver_event_search (rowid, body, room_id, event_id, search_key, sender, event_type) +SELECT stream_ordering, json_extract(event_json, '$.content.body'), room_id, event_id, + 'content.body', sender, event_type +FROM roomserver_events +WHERE event_type = 'm.room.message' AND NOT outlier AND redacted_by = '' + AND json_type(event_json, '$.content.body') = 'text'; + +INSERT INTO roomserver_event_search (rowid, body, room_id, event_id, search_key, sender, event_type) +SELECT stream_ordering, json_extract(event_json, '$.content.name'), room_id, event_id, + 'content.name', sender, event_type +FROM roomserver_events +WHERE event_type = 'm.room.name' AND NOT outlier AND redacted_by = '' + AND json_type(event_json, '$.content.name') = 'text'; + +INSERT INTO roomserver_event_search (rowid, body, room_id, event_id, search_key, sender, event_type) +SELECT stream_ordering, json_extract(event_json, '$.content.topic'), room_id, event_id, + 'content.topic', sender, event_type +FROM roomserver_events +WHERE event_type = 'm.room.topic' AND NOT outlier AND redacted_by = '' + AND json_type(event_json, '$.content.topic') = 'text'; diff --git a/server/internal/storage/sqlite/migrations/0013_thread_notifications.sql b/server/internal/storage/sqlite/migrations/0013_thread_notifications.sql new file mode 100644 index 0000000..f86ee84 --- /dev/null +++ b/server/internal/storage/sqlite/migrations/0013_thread_notifications.sql @@ -0,0 +1,21 @@ +-- Phase 10: notifications counted per thread. +-- +-- thread_id is the thread an event that notified a user belongs to: the +-- root its m.relates_to names when rel_type is m.thread, and '' for an +-- event on the main timeline. A read receipt for a thread clears that +-- thread's rows and a receipt for the main timeline clears the '' rows, +-- which is what lets reading one thread leave the rest of the room unread. +ALTER TABLE push_notifications ADD COLUMN thread_id TEXT NOT NULL DEFAULT ''; + +-- The notifications already stored, from the relation 0012 projected onto +-- their events. A notification whose event has gone stays on the main +-- timeline, where an unthreaded or main receipt still clears it. +UPDATE push_notifications SET thread_id = COALESCE(( + SELECT e.relates_to_id FROM roomserver_events e + WHERE e.event_id = push_notifications.event_id AND e.rel_type = 'm.thread' +), ''); + +-- A threaded receipt marks one thread's rows read; the per-thread counts a +-- sync asks for group a room's unread rows by thread. +CREATE INDEX IF NOT EXISTS push_notifications_thread_idx + ON push_notifications (user_id, room_id, thread_id, is_read, event_pos); diff --git a/server/internal/storage/sqlite/sqlite.go b/server/internal/storage/sqlite/sqlite.go index c6d128c..113f953 100644 --- a/server/internal/storage/sqlite/sqlite.go +++ b/server/internal/storage/sqlite/sqlite.go @@ -49,6 +49,56 @@ var settingsQueries = sqlstore.SettingsQueries{ Delete: `DELETE FROM server_settings WHERE setting_key = ?`, } +// searchDialect is the full-text index on FTS5. The table cannot be +// aliased — MATCH and bm25() take its own name — and its rowid is the +// event's stream ordering. bm25() scores better matches lower, so the rank +// is its negation. +var searchDialect = &sqlstore.SearchDialect{ + Insert: `INSERT INTO roomserver_event_search (rowid, room_id, event_id, search_key, sender, event_type, body) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + DeleteEvent: `DELETE FROM roomserver_event_search WHERE rowid IN ( + SELECT stream_ordering FROM roomserver_events WHERE room_id = ? AND event_id = ?)`, + DeleteRoom: `DELETE FROM roomserver_event_search WHERE room_id = ?`, + From: `roomserver_event_search`, + Name: `roomserver_event_search`, + Position: `roomserver_event_search.rowid`, + Match: `roomserver_event_search MATCH ?`, + Rank: `-bm25(roomserver_event_search)`, + Query: ftsQuery, +} + +// ftsQuery builds an FTS5 query from a search term: every word, quoted so +// that nothing in it is read as an operator, and matched as a prefix so +// that a user who has typed "hel" already finds "hello". Words side by +// side are ANDed. +func ftsQuery(term string) string { + words := storage.SearchWords(term) + for i, w := range words { + words[i] = `"` + strings.ReplaceAll(w, `"`, `""`) + `"*` + } + return strings.Join(words, " ") +} + +// StoreOptions are the sqlstore options for a SQLite database already +// opened on db: the migrations, statements and error recognition every +// SQLite-backed driver shares. The PocketBase driver runs on a SQLite file +// of its own and uses them as they are. +func StoreOptions(db *sql.DB, log *slog.Logger) sqlstore.Options { + return sqlstore.Options{ + DriverName: storage.DriverSQLite, + DB: db, + Dialect: dialect, + MigrationsFS: migrationFiles, + MigrationsDir: "migrations", + Settings: settingsQueries, + // SQLite already uses the `?` placeholders the shared component + // stores are written with, so there is nothing to rebind. + IsUniqueViolation: isUniqueViolation, + Search: searchDialect, + Logger: log, + } +} + // Database is the SQLite implementation of storage.Database. type Database struct { *sqlstore.Store @@ -86,18 +136,7 @@ func Open(ctx context.Context, cfg storage.Config, log *slog.Logger) (storage.Da return nil, fmt.Errorf("sqlite: connect: %w", err) } - store, err := sqlstore.New(sqlstore.Options{ - DriverName: storage.DriverSQLite, - DB: db, - Dialect: dialect, - MigrationsFS: migrationFiles, - MigrationsDir: "migrations", - Settings: settingsQueries, - // SQLite already uses the `?` placeholders the shared component - // stores are written with, so there is nothing to rebind. - IsUniqueViolation: isUniqueViolation, - Logger: log, - }) + store, err := sqlstore.New(StoreOptions(db, log)) if err != nil { _ = db.Close() return nil, err diff --git a/server/internal/storage/sqlstore/push.go b/server/internal/storage/sqlstore/push.go index 743ba4a..e92a69d 100644 --- a/server/internal/storage/sqlstore/push.go +++ b/server/internal/storage/sqlstore/push.go @@ -143,8 +143,8 @@ func (s *Push) DeletePushersForDevices(ctx context.Context, localpart string, de // to NUMERIC, which does not scan into an int64. const ( insertNotificationSQL = `INSERT INTO push_notifications - (id, user_id, room_id, event_id, event_pos, ts_ms, highlight, actions, is_read) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + (id, user_id, room_id, event_id, event_pos, thread_id, ts_ms, highlight, actions, is_read) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` bumpRoomCountsSQL = `INSERT INTO push_room_counts (user_id, room_id, notification_count, highlight_count, stream_pos) @@ -157,6 +157,10 @@ const ( markReadSQL = `UPDATE push_notifications SET is_read = ? WHERE user_id = ? AND room_id = ? AND is_read = ? AND event_pos <= ?` + // markThreadReadSQL is markReadSQL for one thread, or for the main + // timeline as thread ''. + markThreadReadSQL = markReadSQL + ` AND thread_id = ?` + countUnreadSQL = `SELECT COUNT(*), CAST(COALESCE(SUM(CASE WHEN highlight THEN 1 ELSE 0 END), 0) AS BIGINT) FROM push_notifications WHERE user_id = ? AND room_id = ? AND is_read = ?` @@ -174,7 +178,7 @@ const ( selectUnreadTotalSQL = `SELECT CAST(COALESCE(SUM(notification_count), 0) AS BIGINT) FROM push_room_counts WHERE user_id = ?` - notificationColumns = `id, user_id, room_id, event_id, event_pos, ts_ms, highlight, actions, is_read` + notificationColumns = `id, user_id, room_id, event_id, event_pos, thread_id, ts_ms, highlight, actions, is_read` pruneNotificationsSQL = `DELETE FROM push_notifications WHERE is_read = ? AND ts_ms < ?` ) @@ -196,7 +200,7 @@ func (s *Push) AddNotifications(ctx context.Context, ns []storage.NewNotificatio return err } if _, err := tx.ExecContext(ctx, s.q(insertNotificationSQL), pos, n.UserID, n.RoomID, n.EventID, - n.EventPos, n.TimestampMS, n.Highlight, string(n.Actions), false); err != nil { + n.EventPos, n.ThreadID, n.TimestampMS, n.Highlight, string(n.Actions), false); err != nil { return fmt.Errorf("write a notification for %q: %w", n.UserID, err) } highlight := 0 @@ -218,14 +222,19 @@ func (s *Push) AddNotifications(ctx context.Context, ns []storage.NewNotificatio // The counts are recounted from the rows that are still unread rather than // decremented, so that whatever got them out of step — a crash between // two writes of an older version, a row pruned by hand — is put right by -// the next receipt. -func (s *Push) MarkNotificationsRead(ctx context.Context, userID, roomID string, upTo int64) (int64, bool, error) { +// the next receipt. The room's counts stay the room's total across every +// thread whichever thread the receipt read. +func (s *Push) MarkNotificationsRead(ctx context.Context, userID, roomID, thread string, upTo int64) (int64, bool, error) { var ( pos int64 moved bool ) err := s.tx(ctx, func(tx *sql.Tx) error { - res, err := tx.ExecContext(ctx, s.q(markReadSQL), true, userID, roomID, false, upTo) + query, args := markReadSQL, []any{true, userID, roomID, false, upTo} + if thread != storage.AllThreads { + query, args = markThreadReadSQL, append(args, thread) + } + res, err := tx.ExecContext(ctx, s.q(query), args...) if err != nil { return fmt.Errorf("mark %q's notifications read: %w", userID, err) } @@ -291,6 +300,53 @@ func (s *Push) NotificationCounts(ctx context.Context, userID string, roomIDs [] return out, nil } +// ThreadNotificationCounts implements storage.PushStore. +// +// These are counted from the unread rows rather than kept like the room +// totals: a room has few unread notifications at a time, they are a range +// of an index, and a table of per-thread counts would be one more thing +// every write and every receipt has to keep in step. +func (s *Push) ThreadNotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]map[string]storage.NotificationCounts, error) { + out := make(map[string]map[string]storage.NotificationCounts, len(roomIDs)) + if len(roomIDs) == 0 { + return out, nil + } + + query := `SELECT room_id, thread_id, COUNT(*), + CAST(COALESCE(SUM(CASE WHEN highlight THEN 1 ELSE 0 END), 0) AS BIGINT) + FROM push_notifications + WHERE user_id = ? AND is_read = ? AND room_id IN (` + placeholders(len(roomIDs)) + `) + GROUP BY room_id, thread_id` + args := make([]any, 0, len(roomIDs)+2) + args = append(args, userID, false) + for _, id := range roomIDs { + args = append(args, id) + } + rows, err := s.db.QueryContext(ctx, s.q(query), args...) + if err != nil { + return nil, fmt.Errorf("read %q's thread notification counts: %w", userID, err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var roomID, thread string + var notifications, highlights int64 + if err := rows.Scan(&roomID, &thread, ¬ifications, &highlights); err != nil { + return nil, fmt.Errorf("read %q's thread notification counts: %w", userID, err) + } + if out[roomID] == nil { + out[roomID] = map[string]storage.NotificationCounts{} + } + out[roomID][thread] = storage.NotificationCounts{ + Notifications: int(notifications), + Highlights: int(highlights), + } + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read %q's thread notification counts: %w", userID, err) + } + return out, nil +} + // NotificationCountChanges implements storage.PushStore. func (s *Push) NotificationCountChanges(ctx context.Context, userID string, from, to int64) ([]string, error) { rows, err := s.db.QueryContext(ctx, s.q(selectCountChangesSQL), userID, from, to) @@ -345,7 +401,7 @@ func (s *Push) Notifications(ctx context.Context, userID string, before int64, l for rows.Next() { var n storage.Notification var actions string - if err := rows.Scan(&n.ID, &n.UserID, &n.RoomID, &n.EventID, &n.EventPos, &n.TimestampMS, + if err := rows.Scan(&n.ID, &n.UserID, &n.RoomID, &n.EventID, &n.EventPos, &n.ThreadID, &n.TimestampMS, &n.Highlight, &actions, &n.Read); err != nil { return nil, fmt.Errorf("read %q's notifications: %w", userID, err) } diff --git a/server/internal/storage/sqlstore/relations.go b/server/internal/storage/sqlstore/relations.go new file mode 100644 index 0000000..d9c9682 --- /dev/null +++ b/server/internal/storage/sqlstore/relations.go @@ -0,0 +1,495 @@ +package sqlstore + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// The relation and search half of the room store: the reads behind +// /relations, /threads, the bundled summaries in unsigned.m.relations, and +// /search. +// +// Relations need nothing but the two columns on roomserver_events and their +// index, and are written once for both engines like everything else here. +// Search is the one place the engines really differ — SQLite has FTS5, +// PostgreSQL has tsvector — so the statements that touch the index come +// from the driver as a SearchDialect, and only the shape of the query is +// shared. + +// SearchDialect is one engine's full-text index. +// +// Every statement is written against the table roomserver_event_search, +// referred to as Name where it is read, with `?` placeholders that are +// rebound like any other statement. +type SearchDialect struct { + // Insert adds one event, taking (stream_ordering, room_id, event_id, + // search_key, sender, event_type, body) in that order. + Insert string + + // DeleteEvent removes an event's row, taking (room_id, event_id). + DeleteEvent string + + // DeleteRoom removes a room's rows, taking (room_id). + DeleteRoom string + + // From is the FROM clause the search reads, and Name what its columns + // are qualified with. SQLite's FTS5 cannot be aliased — MATCH and + // bm25() take the table's own name — so the two are separate. + From string + Name string + + // Position is the column holding a row's stream ordering. + Position string + + // Match is the condition selecting the rows that match. It takes the + // engine query once. + Match string + + // Rank is the relevance of a matching row, higher being better. It + // takes the engine query RankArgs times. + Rank string + RankArgs int + + // Query turns what a user typed into the engine's query syntax. It + // returns "" for a term with nothing searchable in it, which matches + // nothing. + Query func(term string) string +} + +// WithSearch sets the full-text index the store writes and reads. +func (s *Rooms) WithSearch(d *SearchDialect) *Rooms { + s.search = d + return s +} + +// indexEvent adds an event that has just joined a timeline to the search +// index, if it has anything to find it by. +func (s *Rooms) indexEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEvent, ordering int64) error { + if s.search == nil || e.SearchKey == "" || strings.TrimSpace(e.SearchText) == "" { + return nil + } + _, err := tx.ExecContext(ctx, s.q(s.search.Insert), + ordering, e.RoomID, e.EventID, e.SearchKey, e.Sender, e.Type, e.SearchText) + if err != nil { + return fmt.Errorf("index event %s for search: %w", e.EventID, err) + } + return nil +} + +// unindexEvent removes an event from the search index. +func (s *Rooms) unindexEvent(ctx context.Context, tx *sql.Tx, roomID, eventID string) error { + if s.search == nil { + return nil + } + if _, err := tx.ExecContext(ctx, s.q(s.search.DeleteEvent), roomID, eventID); err != nil { + return fmt.Errorf("remove event %s from the search index: %w", eventID, err) + } + return nil +} + +// Relations implements storage.RoomStore. +func (s *Rooms) Relations(ctx context.Context, req storage.RelationsRequest) ([]storage.StoredEvent, error) { + if req.Limit <= 0 || len(req.ParentIDs) == 0 { + return nil, nil + } + + conditions := []string{ + `e.room_id = ?`, + `e.relates_to_id IN (` + placeholders(len(req.ParentIDs)) + `)`, + `NOT e.outlier`, + } + args := []any{req.RoomID} + for _, id := range req.ParentIDs { + args = append(args, id) + } + if req.RelType != "" { + conditions = append(conditions, `e.rel_type = ?`) + args = append(args, req.RelType) + } else { + // A parent's own row has relates_to_id '' and can never match an + // IN list of event IDs, but '' itself could be passed; it names no + // event and has no children. + conditions = append(conditions, `e.rel_type <> ''`) + } + if req.EventType != "" { + conditions = append(conditions, `e.event_type = ?`) + args = append(args, req.EventType) + } + if req.Low != 0 { + conditions = append(conditions, `e.stream_ordering > ?`) + args = append(args, req.Low) + } + if req.High != 0 { + conditions = append(conditions, `e.stream_ordering <= ?`) + args = append(args, req.High) + } + order := `ASC` + if req.Direction == storage.DirectionBackward { + order = `DESC` + } + args = append(args, req.Limit) + + query := `SELECT ` + eventColumns + ` + FROM roomserver_events e + WHERE ` + strings.Join(conditions, " AND ") + ` + ORDER BY e.stream_ordering ` + order + ` + LIMIT ?` + rows, err := s.db.QueryContext(ctx, s.q(query), args...) + if err != nil { + return nil, fmt.Errorf("read the relations of %v in %q: %w", req.ParentIDs, req.RoomID, err) + } + events, err := scanEvents(rows) + if err != nil { + return nil, fmt.Errorf("read the relations of %v in %q: %w", req.ParentIDs, req.RoomID, err) + } + return events, nil +} + +// ChildEventIDs implements storage.RoomStore. +func (s *Rooms) ChildEventIDs(ctx context.Context, roomID string, parentIDs []string, relType, eventType string) ([]string, error) { + if len(parentIDs) == 0 { + return nil, nil + } + query := `SELECT event_id FROM roomserver_events + WHERE room_id = ? AND relates_to_id IN (` + placeholders(len(parentIDs)) + `) + AND rel_type <> '' AND NOT outlier` + args := []any{roomID} + for _, id := range parentIDs { + args = append(args, id) + } + if relType != "" { + query += ` AND rel_type = ?` + args = append(args, relType) + } + if eventType != "" { + query += ` AND event_type = ?` + args = append(args, eventType) + } + query += ` ORDER BY stream_ordering` + + ids, err := s.queryStrings(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("read the children of %v in %q: %w", parentIDs, roomID, err) + } + return ids, nil +} + +// RelationAggregates implements storage.RoomStore. +// +// It groups by parent, relation type and sender, and joins each group's +// newest position back onto the events table for its event ID: a portable +// spelling of "the row with the largest stream ordering in each group", +// which neither engine can say in one aggregate. +func (s *Rooms) RelationAggregates(ctx context.Context, roomID string, parentIDs []string, high int64) ([]storage.RelationAggregate, error) { + if len(parentIDs) == 0 { + return nil, nil + } + inner := `SELECT relates_to_id, rel_type, sender, COUNT(*) AS n, MAX(stream_ordering) AS latest + FROM roomserver_events + WHERE room_id = ? AND relates_to_id IN (` + placeholders(len(parentIDs)) + `) + AND rel_type IN (?, ?, ?) AND NOT outlier` + args := []any{roomID} + for _, id := range parentIDs { + args = append(args, id) + } + args = append(args, storage.RelThread, storage.RelReplace, storage.RelReference) + if high != 0 { + inner += ` AND stream_ordering <= ?` + args = append(args, high) + } + inner += ` GROUP BY relates_to_id, rel_type, sender` + + query := `SELECT g.relates_to_id, g.rel_type, g.sender, g.n, g.latest, e.event_id + FROM (` + inner + `) g + JOIN roomserver_events e ON e.stream_ordering = g.latest` + rows, err := s.db.QueryContext(ctx, s.q(query), args...) + if err != nil { + return nil, fmt.Errorf("summarise the relations of %d events in %q: %w", len(parentIDs), roomID, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.RelationAggregate + for rows.Next() { + var a storage.RelationAggregate + if err := rows.Scan(&a.ParentID, &a.RelType, &a.Sender, &a.Count, &a.LatestOrdering, &a.LatestEventID); err != nil { + return nil, fmt.Errorf("summarise the relations in %q: %w", roomID, err) + } + out = append(out, a) + } + return out, rows.Err() +} + +// RelationEventIDs implements storage.RoomStore. +// +// The limit is per parent in spirit and applied over the whole set: the +// summary it feeds is a sample, and a parent with more children than the +// limit is rare enough that one query beats one per parent. +func (s *Rooms) RelationEventIDs(ctx context.Context, roomID string, parentIDs []string, relType string, high int64, limit int) (map[string][]string, error) { + if len(parentIDs) == 0 || limit <= 0 { + return nil, nil + } + query := `SELECT relates_to_id, event_id FROM roomserver_events + WHERE room_id = ? AND relates_to_id IN (` + placeholders(len(parentIDs)) + `) + AND rel_type = ? AND NOT outlier` + args := []any{roomID} + for _, id := range parentIDs { + args = append(args, id) + } + args = append(args, relType) + if high != 0 { + query += ` AND stream_ordering <= ?` + args = append(args, high) + } + query += ` ORDER BY stream_ordering LIMIT ?` + args = append(args, limit*len(parentIDs)) + + rows, err := s.db.QueryContext(ctx, s.q(query), args...) + if err != nil { + return nil, fmt.Errorf("read the %s relations in %q: %w", relType, roomID, err) + } + defer func() { _ = rows.Close() }() + + out := map[string][]string{} + for rows.Next() { + var parent, id string + if err := rows.Scan(&parent, &id); err != nil { + return nil, fmt.Errorf("read the %s relations in %q: %w", relType, roomID, err) + } + if len(out[parent]) < limit { + out[parent] = append(out[parent], id) + } + } + return out, rows.Err() +} + +// Threads implements storage.RoomStore. +// +// A thread is a root with at least one m.thread child, and is ordered by its +// newest child. The root has to be an event of the room this server holds +// in its timeline: a thread whose root is missing — not yet backfilled, or +// never sent here — cannot be shown as a thread at all. +func (s *Rooms) Threads(ctx context.Context, req storage.ThreadsRequest) ([]storage.Thread, error) { + if req.Limit <= 0 { + return nil, nil + } + inner := `SELECT relates_to_id AS root, MAX(stream_ordering) AS latest + FROM roomserver_events + WHERE room_id = ? AND rel_type = ? AND NOT outlier` + args := []any{req.RoomID, storage.RelThread} + if req.High != 0 { + inner += ` AND stream_ordering <= ?` + args = append(args, req.High) + } + inner += ` GROUP BY relates_to_id` + + conditions := []string{`r.room_id = ?`, `NOT r.outlier`} + args = append(args, req.RoomID) + if req.Before != 0 { + conditions = append(conditions, `g.latest < ?`) + args = append(args, req.Before) + } + if req.ParticipatedBy != "" { + conditions = append(conditions, `(r.sender = ? OR EXISTS ( + SELECT 1 FROM roomserver_events p + WHERE p.room_id = r.room_id AND p.relates_to_id = g.root AND p.rel_type = ? AND p.sender = ?))`) + args = append(args, req.ParticipatedBy, storage.RelThread, req.ParticipatedBy) + } + args = append(args, req.Limit) + + query := `SELECT g.root, g.latest + FROM (` + inner + `) g + JOIN roomserver_events r ON r.event_id = g.root + WHERE ` + strings.Join(conditions, " AND ") + ` + ORDER BY g.latest DESC + LIMIT ?` + rows, err := s.db.QueryContext(ctx, s.q(query), args...) + if err != nil { + return nil, fmt.Errorf("read the threads of %q: %w", req.RoomID, err) + } + defer func() { _ = rows.Close() }() + + var out []storage.Thread + for rows.Next() { + var t storage.Thread + if err := rows.Scan(&t.RootID, &t.LatestOrdering); err != nil { + return nil, fmt.Errorf("read the threads of %q: %w", req.RoomID, err) + } + out = append(out, t) + } + return out, rows.Err() +} + +// searchKeys are the fields the index holds. +var searchKeys = []string{"content.body", "content.name", "content.topic"} + +// Search implements storage.RoomStore. +func (s *Rooms) Search(ctx context.Context, req storage.SearchRequest) ([]storage.SearchResult, int, error) { + if s.search == nil || req.Limit <= 0 || len(req.Rooms) == 0 { + return nil, 0, nil + } + term := s.search.Query(req.Term) + if term == "" { + return nil, 0, nil + } + + pos := s.search.Position + col := func(name string) string { return s.search.Name + "." + name } + conditions := []string{s.search.Match} + args := []any{term} + + // One bounded clause per room: what a user may search of a room is a + // window of its stream, and the windows differ from room to room. + rooms := make([]string, 0, len(req.Rooms)) + for _, r := range req.Rooms { + clause := col(`room_id`) + ` = ?` + args = append(args, r.RoomID) + if r.Low != 0 { + clause += ` AND ` + pos + ` > ?` + args = append(args, r.Low) + } + if r.High != 0 { + clause += ` AND ` + pos + ` <= ?` + args = append(args, r.High) + } + rooms = append(rooms, `(`+clause+`)`) + } + conditions = append(conditions, `(`+strings.Join(rooms, ` OR `)+`)`) + + keys := req.Keys + if len(keys) == 0 { + keys = searchKeys + } + conditions = append(conditions, col(`search_key`)+` IN (`+placeholders(len(keys))+`)`) + for _, k := range keys { + args = append(args, k) + } + + if clause, filterArgs := filterConditionsOn(col(`event_type`), col(`sender`), req.Filter); clause != "" { + conditions = append(conditions, clause) + args = append(args, filterArgs...) + } + where := strings.Join(conditions, " AND ") + + var total int + countQuery := `SELECT COUNT(*) FROM ` + s.search.From + ` WHERE ` + where + if err := s.db.QueryRowContext(ctx, s.q(countQuery), args...).Scan(&total); err != nil { + return nil, 0, fmt.Errorf("count search results: %w", err) + } + + pageWhere := where + pageArgs := append([]any{}, args...) + if req.Order == storage.SearchByRecent && req.Before != 0 { + pageWhere += ` AND ` + pos + ` < ?` + pageArgs = append(pageArgs, req.Before) + } + + rankArgs := make([]any, 0, s.search.RankArgs) + for range s.search.RankArgs { + rankArgs = append(rankArgs, term) + } + order := `rank DESC, ` + pos + ` DESC` + if req.Order == storage.SearchByRecent { + order = pos + ` DESC` + } + query := `SELECT ` + pos + `, ` + s.search.Rank + ` AS rank + FROM ` + s.search.From + ` + WHERE ` + pageWhere + ` + ORDER BY ` + order + ` + LIMIT ? OFFSET ?` + offset := req.Offset + if req.Order == storage.SearchByRecent { + offset = 0 + } + queryArgs := append(rankArgs, pageArgs...) + queryArgs = append(queryArgs, req.Limit, offset) + + hits, err := s.searchHits(ctx, query, queryArgs) + if err != nil { + return nil, 0, err + } + if len(hits) == 0 { + return nil, total, nil + } + + // The events themselves, read by position in one query and put back in + // the order the search ranked them. + eventArgs := make([]any, 0, len(hits)) + for _, h := range hits { + eventArgs = append(eventArgs, h.pos) + } + eventRows, err := s.db.QueryContext(ctx, s.q(`SELECT `+eventColumns+` + FROM roomserver_events e WHERE e.stream_ordering IN (`+placeholders(len(hits))+`)`), eventArgs...) + if err != nil { + return nil, 0, fmt.Errorf("read search results: %w", err) + } + events, err := scanEvents(eventRows) + if err != nil { + return nil, 0, fmt.Errorf("read search results: %w", err) + } + byPos := make(map[int64]storage.StoredEvent, len(events)) + for _, e := range events { + byPos[e.StreamOrdering] = e + } + + out := make([]storage.SearchResult, 0, len(hits)) + for _, h := range hits { + // An index row whose event has gone — purged under a search in + // flight — is skipped rather than reported. + if e, ok := byPos[h.pos]; ok { + out = append(out, storage.SearchResult{Event: e, Rank: h.rank}) + } + } + return out, total, nil +} + +// searchHit is one row of a search page: where the event is, and how well +// it matched. +type searchHit struct { + pos int64 + rank float64 +} + +// searchHits runs the page query of a search. +func (s *Rooms) searchHits(ctx context.Context, query string, args []any) ([]searchHit, error) { + rows, err := s.db.QueryContext(ctx, s.q(query), args...) + if err != nil { + return nil, fmt.Errorf("search: %w", err) + } + defer func() { _ = rows.Close() }() + + var hits []searchHit + for rows.Next() { + var h searchHit + if err := rows.Scan(&h.pos, &h.rank); err != nil { + return nil, fmt.Errorf("search: %w", err) + } + hits = append(hits, h) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("search: %w", err) + } + return hits, nil +} + +// queryStrings runs a query returning one string column. +func (s *Rooms) queryStrings(ctx context.Context, query string, args ...any) ([]string, error) { + rows, err := s.db.QueryContext(ctx, s.q(query), args...) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var out []string + for rows.Next() { + var v string + if err := rows.Scan(&v); err != nil { + return nil, err + } + out = append(out, v) + } + return out, rows.Err() +} diff --git a/server/internal/storage/sqlstore/rooms.go b/server/internal/storage/sqlstore/rooms.go index ef3738a..05d9523 100644 --- a/server/internal/storage/sqlstore/rooms.go +++ b/server/internal/storage/sqlstore/rooms.go @@ -25,6 +25,9 @@ type Rooms struct { db *sql.DB rebind func(string) string isUnique func(error) bool + + // search is the engine's full-text index, nil for none. + search *SearchDialect } // NewRooms builds a room store. @@ -266,9 +269,10 @@ const ( insertEventSQL = ` INSERT INTO roomserver_events ( event_id, room_id, stream_ordering, event_type, state_key, - sender, depth, origin_server_ts, event_json, state_before_id, outlier) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - selectOutlierSQL = `SELECT outlier FROM roomserver_events WHERE event_id = ?` + sender, depth, origin_server_ts, event_json, state_before_id, outlier, + relates_to_id, rel_type) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + selectOutlierSQL = `SELECT outlier, redacted_by FROM roomserver_events WHERE event_id = ?` promoteOutlierSQL = ` UPDATE roomserver_events SET stream_ordering = ?, state_before_id = ?, outlier = ? @@ -284,8 +288,11 @@ const ( // the event ID says the two copies are the same event, and the stored one // may already have been redacted. func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEvent, ordering, snapshotID int64, outlier bool) error { - var existingOutlier bool - err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), e.EventID).Scan(&existingOutlier) + var ( + existingOutlier bool + redactedBy string + ) + err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), e.EventID).Scan(&existingOutlier, &redactedBy) switch { case err == nil: if existingOutlier && !outlier { @@ -293,6 +300,13 @@ func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEve ordering, snapshotID, false, e.EventID, true); err != nil { return fmt.Errorf("promote outlier: %w", err) } + // It joins the timeline, so it joins the search index — unless + // the copy already held was redacted, whose text is gone. + if redactedBy == "" { + if err := s.indexEvent(ctx, tx, e, ordering); err != nil { + return err + } + } } // The same event twice is not a failure worth propagating: an // event ID is a hash of its contents, so the second copy says @@ -304,13 +318,17 @@ func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEve _, err = tx.ExecContext(ctx, s.q(insertEventSQL), e.EventID, e.RoomID, ordering, e.Type, e.StateKey, - e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, outlier) + e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, outlier, + e.RelatesTo, e.RelType) if err != nil { if s.isUnique(err) { return nil } return fmt.Errorf("insert event: %w", err) } + if !outlier { + return s.indexEvent(ctx, tx, e, ordering) + } return nil } @@ -403,9 +421,13 @@ func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []stora // which is already in the timeline and left alone. var write []storage.BackfillEvent promote := map[string]bool{} + redacted := map[string]bool{} for _, b := range events { - var outlier bool - err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), b.Event.EventID).Scan(&outlier) + var ( + outlier bool + redactedBy string + ) + err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), b.Event.EventID).Scan(&outlier, &redactedBy) switch { case errors.Is(err, sql.ErrNoRows): write = append(write, b) @@ -414,6 +436,7 @@ func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []stora case outlier: write = append(write, b) promote[b.Event.EventID] = true + redacted[b.Event.EventID] = redactedBy != "" } } if len(write) == 0 { @@ -439,13 +462,22 @@ func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []stora ordering, snapshotID, false, e.EventID, true); err != nil { return fmt.Errorf("promote outlier %s: %w", e.EventID, err) } + if !redacted[e.EventID] { + if err := s.indexEvent(ctx, tx, e, ordering); err != nil { + return err + } + } continue } if _, err := tx.ExecContext(ctx, s.q(insertEventSQL), e.EventID, e.RoomID, ordering, e.Type, e.StateKey, - e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, false); err != nil { + e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, false, + e.RelatesTo, e.RelType); err != nil { return fmt.Errorf("insert event %s: %w", e.EventID, err) } + if err := s.indexEvent(ctx, tx, e, ordering); err != nil { + return err + } } return nil }) @@ -456,7 +488,7 @@ func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []stora } const applyRedactionSQL = ` - UPDATE roomserver_events SET event_json = ?, redacted_by = ? + UPDATE roomserver_events SET event_json = ?, redacted_by = ?, relates_to_id = '', rel_type = '' WHERE room_id = ? AND event_id = ? AND redacted_by = ''` // applyRedaction replaces a stored event with its redacted form. @@ -470,7 +502,9 @@ func (s *Rooms) applyRedaction(ctx context.Context, tx *sql.Tx, roomID, targetID if err != nil { return fmt.Errorf("apply redaction to %s: %w", targetID, err) } - return nil + // What a redacted event said is gone, and so is its way of being found + // by what it said. + return s.unindexEvent(ctx, tx, roomID, targetID) } const ( @@ -574,7 +608,8 @@ func (s *Rooms) recordTransaction(ctx context.Context, tx *sql.Tx, ref storage.T // scanEvent expects. const eventColumns = ` e.room_id, e.event_id, e.event_type, e.state_key, e.sender, - e.depth, e.origin_server_ts, e.event_json, e.stream_ordering, e.redacted_by, e.outlier` + e.depth, e.origin_server_ts, e.event_json, e.stream_ordering, e.redacted_by, e.outlier, + e.relates_to_id, e.rel_type` // scanEvent reads one row of eventColumns. func scanEvent(scan func(...any) error) (storage.StoredEvent, error) { @@ -584,7 +619,8 @@ func scanEvent(scan func(...any) error) (storage.StoredEvent, error) { body string ) err := scan(&e.RoomID, &e.EventID, &e.Type, &stateKey, &e.Sender, - &e.Depth, &e.OriginServerTS, &body, &e.StreamOrdering, &e.RedactedBy, &e.Outlier) + &e.Depth, &e.OriginServerTS, &body, &e.StreamOrdering, &e.RedactedBy, &e.Outlier, + &e.RelatesTo, &e.RelType) if err != nil { return storage.StoredEvent{}, err } @@ -1087,6 +1123,13 @@ var purgeStatements = []string{ // PurgeRoom implements storage.RoomStore. func (s *Rooms) PurgeRoom(ctx context.Context, roomID string) error { err := s.tx(ctx, func(tx *sql.Tx) error { + // The search index has no foreign key to cascade from: SQLite's is + // a virtual table. + if s.search != nil { + if _, err := tx.ExecContext(ctx, s.q(s.search.DeleteRoom), roomID); err != nil { + return err + } + } for i, stmt := range purgeStatements { result, err := tx.ExecContext(ctx, s.q(stmt), roomID) if err != nil { diff --git a/server/internal/storage/sqlstore/sqlstore.go b/server/internal/storage/sqlstore/sqlstore.go index 6dcb561..290339f 100644 --- a/server/internal/storage/sqlstore/sqlstore.go +++ b/server/internal/storage/sqlstore/sqlstore.go @@ -45,6 +45,10 @@ type Options struct { // without a check-then-insert race. IsUniqueViolation func(error) bool + // Search is this engine's full-text search. Nil leaves the index + // unwritten and /search finding nothing. + Search *SearchDialect + // Logger is used for migration progress. Logger *slog.Logger } @@ -81,7 +85,7 @@ func New(opts Options) (*Store, error) { log: opts.Logger, settings: NewSettings(opts.DB, opts.Settings), users: NewUsers(opts.DB, opts.Rebind, opts.IsUniqueViolation), - rooms: NewRooms(opts.DB, opts.Rebind, opts.IsUniqueViolation), + rooms: NewRooms(opts.DB, opts.Rebind, opts.IsUniqueViolation).WithSearch(opts.Search), sync: NewSync(opts.DB, opts.Rebind, opts.IsUniqueViolation), fed: NewFederation(opts.DB, opts.Rebind), media: NewMedia(opts.DB, opts.Rebind, opts.IsUniqueViolation), diff --git a/server/internal/storage/sqlstore/timeline.go b/server/internal/storage/sqlstore/timeline.go index 33b512e..a398a6d 100644 --- a/server/internal/storage/sqlstore/timeline.go +++ b/server/internal/storage/sqlstore/timeline.go @@ -183,6 +183,12 @@ func (s *Rooms) StreamEvents(ctx context.Context, after, upTo int64, limit int) // not_types win over types for an event matching both — the behaviour the // spec requires. func filterConditions(f storage.TimelineFilter) (string, []any) { + return filterConditionsOn(`e.event_type`, `e.sender`, f) +} + +// filterConditionsOn is filterConditions against named columns, for a +// table other than roomserver_events. +func filterConditionsOn(typeColumn, senderColumn string, f storage.TimelineFilter) (string, []any) { var ( clauses []string args []any @@ -198,10 +204,10 @@ func filterConditions(f storage.TimelineFilter) (string, []any) { clauses = append(clauses, clause) args = append(args, clauseArgs...) } - add(`e.event_type`, f.Types, false) - add(`e.event_type`, f.NotTypes, true) - add(`e.sender`, f.Senders, false) - add(`e.sender`, f.NotSenders, true) + add(typeColumn, f.Types, false) + add(typeColumn, f.NotTypes, true) + add(senderColumn, f.Senders, false) + add(senderColumn, f.NotSenders, true) if len(clauses) == 0 { return "", nil @@ -299,6 +305,48 @@ func (s *Rooms) RoomsWithEvents(ctx context.Context, roomIDs []string, from, to return out, rows.Err() } +// LatestEventPositions implements storage.RoomStore. +func (s *Rooms) LatestEventPositions(ctx context.Context, roomIDs, types []string, upTo int64) (map[string]int64, error) { + if len(roomIDs) == 0 || len(types) == 0 { + return map[string]int64{}, nil + } + query := ` + SELECT room_id, MAX(stream_ordering) FROM roomserver_events + WHERE room_id IN (` + placeholders(len(roomIDs)) + `) + AND event_type IN (` + placeholders(len(types)) + `) AND NOT outlier` + args := make([]any, 0, len(roomIDs)+len(types)+1) + for _, id := range roomIDs { + args = append(args, id) + } + for _, t := range types { + args = append(args, t) + } + if upTo != 0 { + query += ` AND stream_ordering <= ?` + args = append(args, upTo) + } + query += ` GROUP BY room_id` + + rows, err := s.db.QueryContext(ctx, s.q(query), args...) + if err != nil { + return nil, fmt.Errorf("find the latest activity of %d rooms: %w", len(roomIDs), err) + } + defer func() { _ = rows.Close() }() + + out := make(map[string]int64, len(roomIDs)) + for rows.Next() { + var ( + roomID string + pos int64 + ) + if err := rows.Scan(&roomID, &pos); err != nil { + return nil, fmt.Errorf("find the latest activity of %d rooms: %w", len(roomIDs), err) + } + out[roomID] = pos + } + return out, rows.Err() +} + const selectEventAtStreamSQL = ` SELECT e.event_id, e.event_type, e.state_key, e.state_before_id FROM roomserver_events e diff --git a/server/internal/storage/storage.go b/server/internal/storage/storage.go index d2cbe0c..400677e 100644 --- a/server/internal/storage/storage.go +++ b/server/internal/storage/storage.go @@ -40,6 +40,21 @@ type Config struct { MaxOpenConns int MaxIdleConns int ConnMaxLifetime time.Duration + + // S3 is the bucket a driver with its own file storage puts media in. + // Only the pocketbase driver reads it. + S3 S3Config +} + +// S3Config is an S3-compatible bucket. +type S3Config struct { + Enabled bool + Bucket string + Region string + Endpoint string + AccessKey string + Secret string + ForcePathStyle bool } // Database is the handle every component gets. Component-specific stores diff --git a/server/internal/storage/storage_test.go b/server/internal/storage/storage_test.go index 7ab7334..9df88f0 100644 --- a/server/internal/storage/storage_test.go +++ b/server/internal/storage/storage_test.go @@ -2,9 +2,11 @@ package storage_test import ( "errors" + "io" "os" "path/filepath" "slices" + "strings" "testing" "github.com/dariuszjaniak/warp/server/internal/storage" @@ -28,6 +30,8 @@ func openTestDB(t *testing.T, driver string) storage.Database { t.Skip("set WARP_TEST_POSTGRES_DSN to test the postgres driver") } cfg.ConnectionString = dsn + case storage.DriverPocketBase: + cfg.ConnectionString = filepath.Join(t.TempDir(), "pb_data") default: t.Fatalf("no test setup for driver %q", driver) } @@ -40,8 +44,9 @@ func openTestDB(t *testing.T, driver string) storage.Database { return db } -// testDrivers are run by every storage test. -var testDrivers = []string{storage.DriverSQLite, storage.DriverPostgres} +// testDrivers are run by every storage test. PocketBase is here because +// parity with sqlite is its whole contract. +var testDrivers = []string{storage.DriverSQLite, storage.DriverPostgres, storage.DriverPocketBase} func TestDriversRegistered(t *testing.T) { got := storage.Drivers() @@ -59,13 +64,56 @@ func TestOpenUnknownDriver(t *testing.T) { } } -func TestPocketBaseIsNotImplemented(t *testing.T) { - _, err := storage.Open(t.Context(), storage.Config{ - Driver: storage.DriverPocketBase, - ConnectionString: "./pb_data", - }, nil) - if !errors.Is(err, storage.ErrNotImplemented) { - t.Fatalf("expected ErrNotImplemented, got %v", err) +func TestPocketBaseNeedsADataDirectory(t *testing.T) { + if _, err := storage.Open(t.Context(), storage.Config{Driver: storage.DriverPocketBase}, nil); err == nil { + t.Fatal("expected an error for a pocketbase driver with no data directory") + } +} + +func TestPocketBaseKeepsMediaInItsFilesystem(t *testing.T) { + db := openTestDB(t, storage.DriverPocketBase) + provider, ok := db.(storage.BlobProvider) + if !ok { + t.Fatal("the pocketbase driver does not provide its own blob store") + } + blobs := provider.Blobs() + ctx := t.Context() + + info, err := blobs.Put(ctx, strings.NewReader("hello media"), 1024) + if err != nil { + t.Fatalf("Put: %v", err) + } + if info.Size != 11 || len(info.SHA256) != 64 { + t.Fatalf("Put returned %+v", info) + } + if again, err := blobs.Put(ctx, strings.NewReader("hello media"), 1024); err != nil || again != info { + t.Errorf("the same content again = %+v, %v", again, err) + } + if _, err := blobs.Put(ctx, strings.NewReader("too long"), 3); !errors.Is(err, storage.ErrBlobTooLarge) { + t.Errorf("an oversize Put = %v, want ErrBlobTooLarge", err) + } + + r, err := blobs.Open(ctx, info.SHA256) + if err != nil { + t.Fatalf("Open: %v", err) + } + body, err := io.ReadAll(r) + _ = r.Close() + if err != nil || string(body) != "hello media" { + t.Errorf("read back %q, %v", body, err) + } + + if err := blobs.Delete(ctx, info.SHA256); err != nil { + t.Fatalf("Delete: %v", err) + } + if err := blobs.Delete(ctx, info.SHA256); err != nil { + t.Errorf("deleting twice = %v", err) + } + if _, err := blobs.Open(ctx, info.SHA256); !errors.Is(err, storage.ErrNoSuchBlob) { + t.Errorf("Open after Delete = %v, want ErrNoSuchBlob", err) + } + if _, err := blobs.Open(ctx, "../../etc/passwd"); !errors.Is(err, storage.ErrNoSuchBlob) { + t.Errorf("Open of a non-hash = %v", err) } } diff --git a/server/internal/syncapi/filter.go b/server/internal/syncapi/filter.go index 4adcc24..4288ed1 100644 --- a/server/internal/syncapi/filter.go +++ b/server/internal/syncapi/filter.go @@ -98,8 +98,10 @@ type RoomEventFilter struct { // this were set; see roomserver.shapeState. IncludeRedundantMembers bool `json:"include_redundant_members,omitempty"` - // UnreadThreadNotifications is a threads feature; threads are - // PLAN.md phase 10. + // UnreadThreadNotifications splits a joined room's unread counts by + // thread: unread_notifications counts the main timeline only, and + // unread_thread_notifications has the counts of each thread. Only the + // room timeline's flag is read, which is where the spec puts it. UnreadThreadNotifications bool `json:"unread_thread_notifications,omitempty"` } @@ -221,6 +223,12 @@ func (f Filter) LazyLoadMembers() bool { return f.Room != nil && f.Room.Timeline != nil && f.Room.Timeline.LazyLoadMembers } +// UnreadThreadNotifications reports whether the filter asks for unread +// counts per thread. +func (f Filter) UnreadThreadNotifications() bool { + return f.Room != nil && f.Room.Timeline != nil && f.Room.Timeline.UnreadThreadNotifications +} + // IncludeLeave reports whether the rooms the user has left should be // reported in full rather than only when the leave is new. func (f Filter) IncludeLeave() bool { diff --git a/server/internal/syncapi/notifications.go b/server/internal/syncapi/notifications.go index 976959e..7b1a6ff 100644 --- a/server/internal/syncapi/notifications.go +++ b/server/internal/syncapi/notifications.go @@ -3,6 +3,8 @@ package syncapi import ( "context" "slices" + + "github.com/dariuszjaniak/warp/server/internal/storage" ) // pushRulesType is the account data push rules travel as. The push server @@ -24,7 +26,15 @@ type UnreadNotifications struct { // counts moved: a room is in a response because something in it changed, // and a client showing a new message wants the count beside it whether or // not this particular sync is the one that moved it. -func (s *API) unreadNotifications(ctx context.Context, userID string, resp *Response) error { +// +// With byThread — the filter's unread_thread_notifications — the counts +// are split: unread_notifications is the main timeline's, and each thread +// with something unread is in unread_thread_notifications. A thread read +// to the end drops out of the map rather than being reported as zero, +// which the spec allows and a client takes the same way: the room is in +// the response because its counts moved, and the map it is sent stands +// for all of the room's threads, so a thread missing from it is read. +func (s *API) unreadNotifications(ctx context.Context, userID string, byThread bool, resp *Response) error { if s.push == nil || len(resp.Rooms.Join) == 0 { return nil } @@ -34,6 +44,35 @@ func (s *API) unreadNotifications(ctx context.Context, userID string, resp *Resp } slices.Sort(roomIDs) + if byThread { + counts, err := s.push.ThreadNotificationCounts(ctx, userID, roomIDs) + if err != nil { + return err + } + for _, roomID := range roomIDs { + room := resp.Rooms.Join[roomID] + main := counts[roomID][storage.ThreadMain] + room.UnreadNotifications = &UnreadNotifications{ + NotificationCount: main.Notifications, + HighlightCount: main.Highlights, + } + for thread, c := range counts[roomID] { + if thread == storage.ThreadMain { + continue + } + if room.UnreadThreadNotifications == nil { + room.UnreadThreadNotifications = map[string]UnreadNotifications{} + } + room.UnreadThreadNotifications[thread] = UnreadNotifications{ + NotificationCount: c.Notifications, + HighlightCount: c.Highlights, + } + } + resp.Rooms.Join[roomID] = room + } + return nil + } + counts, err := s.push.NotificationCounts(ctx, userID, roomIDs) if err != nil { return err diff --git a/server/internal/syncapi/relations.go b/server/internal/syncapi/relations.go new file mode 100644 index 0000000..af0eaee --- /dev/null +++ b/server/internal/syncapi/relations.go @@ -0,0 +1,197 @@ +package syncapi + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// GET /rooms/{roomID}/relations/{eventID} and GET /rooms/{roomID}/threads: +// the children of an event, and the threads of a room. +// +// The reads are the roomserver's (relations.go there); what is here is the +// token handling, which is this package's. Relations page exactly like +// /messages and take the same tokens — a client may start paging an +// event's children from a sync's prev_batch — so they use the same +// stream-position token. Threads are ordered by their latest activity +// rather than by any one stream, and have a token of their own. + +// DefaultRelationsLimit is the page size of /relations and /threads when +// the client names none. +const DefaultRelationsLimit = 50 + +// RelationsRequest is one page of an event's children. +type RelationsRequest struct { + RoomID string + EventID string + User string + + RelType string + EventType string + + // From and To are pagination tokens, as for /messages. + From string + To string + + // Direction defaults to backwards, newest first, as the spec has it. + Direction storage.TimelineDirection + + Limit int + Recurse bool +} + +// RelationsResponse is one page of an event's children. +type RelationsResponse struct { + Chunk []roomserver.ClientEvent `json:"chunk"` + + // NextBatch continues in the direction asked for; it is left out at + // the end. + NextBatch string `json:"next_batch,omitempty"` + + // PrevBatch is the token this page began at, when the client gave one. + PrevBatch string `json:"prev_batch,omitempty"` + + // RecursionDepth is how deep a recursive read went. It is only there + // when the client asked to recurse, which is how it learns the server + // honoured the request. + RecursionDepth int `json:"recursion_depth,omitempty"` +} + +// Relations reads one page of an event's children. +func (s *API) Relations(ctx context.Context, req RelationsRequest) (RelationsResponse, error) { + from, err := ParseToken(req.From) + if err != nil { + return RelationsResponse{}, errInvalidTokenf("from: %v", err) + } + to, err := ParseToken(req.To) + if err != nil { + return RelationsResponse{}, errInvalidTokenf("to: %v", err) + } + direction := req.Direction + if direction != storage.DirectionForward { + direction = storage.DirectionBackward + } + limit := req.Limit + if limit <= 0 { + limit = DefaultRelationsLimit + } + limit = min(limit, MaxTimelineLimit) + + low, high := from.PDU, to.PDU + if direction == storage.DirectionBackward { + low, high = to.PDU, from.PDU + } + + page, err := s.rooms.Relations(ctx, roomserver.RelationsRequest{ + RoomID: req.RoomID, + EventID: req.EventID, + User: req.User, + RelType: req.RelType, + EventType: req.EventType, + Low: low, + High: high, + Direction: direction, + Limit: limit, + Recurse: req.Recurse, + }) + if err != nil { + return RelationsResponse{}, err + } + + chunk := page.Events + if direction == storage.DirectionBackward { + chunk = reversed(chunk) + } + if chunk == nil { + chunk = []roomserver.ClientEvent{} + } + out := RelationsResponse{Chunk: chunk} + if req.From != "" { + out.PrevBatch = req.From + } + if page.Limited { + if direction == storage.DirectionBackward { + out.NextBatch = pduToken(page.Start).String() + } else { + out.NextBatch = pduToken(page.End).String() + } + } + if req.Recurse { + out.RecursionDepth = page.Depth + } + return out, nil +} + +// ThreadsRequest is one page of a room's threads. +type ThreadsRequest struct { + RoomID string + User string + + // Include is "all" or "participated". + Include string + + // From is a token from a previous page's next_batch. + From string + + Limit int +} + +// ThreadsResponse is one page of a room's threads. +type ThreadsResponse struct { + Chunk []roomserver.ClientEvent `json:"chunk"` + NextBatch string `json:"next_batch,omitempty"` +} + +// threadTokenPrefix marks a /threads token, which is a position in thread +// activity rather than a sync token. +const threadTokenPrefix = "t" + +// Threads reads one page of a room's threads. +func (s *API) Threads(ctx context.Context, req ThreadsRequest) (ThreadsResponse, error) { + var participated bool + switch req.Include { + case "", "all": + case "participated": + participated = true + default: + return ThreadsResponse{}, errInvalidParamf("include must be \"all\" or \"participated\"") + } + + var before int64 + if req.From != "" { + raw, ok := strings.CutPrefix(req.From, threadTokenPrefix) + value, err := strconv.ParseInt(raw, 10, 64) + if !ok || err != nil { + return ThreadsResponse{}, errInvalidTokenf("%q is not a /threads token", req.From) + } + before = value + } + limit := req.Limit + if limit <= 0 { + limit = DefaultRelationsLimit + } + limit = min(limit, MaxTimelineLimit) + + page, err := s.rooms.Threads(ctx, roomserver.ThreadsRequest{ + RoomID: req.RoomID, + User: req.User, + Participated: participated, + Before: before, + Limit: limit, + }) + if err != nil { + return ThreadsResponse{}, err + } + out := ThreadsResponse{Chunk: page.Roots} + if out.Chunk == nil { + out.Chunk = []roomserver.ClientEvent{} + } + if page.Next != 0 { + out.NextBatch = fmt.Sprintf("%s%d", threadTokenPrefix, page.Next) + } + return out, nil +} diff --git a/server/internal/syncapi/search.go b/server/internal/syncapi/search.go new file mode 100644 index 0000000..046dae7 --- /dev/null +++ b/server/internal/syncapi/search.go @@ -0,0 +1,362 @@ +package syncapi + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" +) + +// POST /search: full-text search of the rooms a user is or was in. +// +// The roomserver decides where the user may look and the store runs the +// query; what is here is the request and response shape, which carries a +// fair amount besides the hits — the events around each one, the profiles of +// their senders, the rooms' state, and the hits grouped by room or sender — +// all of it optional and all of it what a client needs to show a result +// without a second round of requests. + +// DefaultSearchLimit is the page size when the filter names none. +const DefaultSearchLimit = 10 + +// MaxSearchLimit caps a page. Every hit can bring its context, its +// senders' profiles and its room's state with it, so a page is far heavier +// than its count suggests. +const MaxSearchLimit = 100 + +// MaxSearchContext caps before_limit and after_limit. +const MaxSearchContext = 20 + +// SearchRequest is the body of a /search, with the caller. +type SearchRequest struct { + User string + + // NextBatch is the ?next_batch= of a later page. + NextBatch string + + Body SearchBody +} + +// SearchBody is the request body. +type SearchBody struct { + SearchCategories struct { + RoomEvents *RoomEventsCriteria `json:"room_events"` + } `json:"search_categories"` +} + +// RoomEventsCriteria is the one search category the spec defines. +type RoomEventsCriteria struct { + SearchTerm string `json:"search_term"` + Keys []string `json:"keys,omitempty"` + Filter *RoomEventFilter `json:"filter,omitempty"` + OrderBy string `json:"order_by,omitempty"` + EventContext *struct { + BeforeLimit *int `json:"before_limit,omitempty"` + AfterLimit *int `json:"after_limit,omitempty"` + IncludeProfile bool `json:"include_profile,omitempty"` + } `json:"event_context,omitempty"` + IncludeState bool `json:"include_state,omitempty"` + Groupings *struct { + GroupBy []struct { + Key string `json:"key"` + } `json:"group_by"` + } `json:"groupings,omitempty"` +} + +// SearchResponse is the body of the answer. +type SearchResponse struct { + SearchCategories struct { + RoomEvents RoomEventsResults `json:"room_events"` + } `json:"search_categories"` +} + +// RoomEventsResults is one page of hits. +type RoomEventsResults struct { + Count int `json:"count"` + Highlights []string `json:"highlights"` + Results []SearchHit `json:"results"` + NextBatch string `json:"next_batch,omitempty"` + State map[string][]roomserver.ClientEvent `json:"state,omitempty"` + Groups map[string]map[string]SearchGroupEntry `json:"groups,omitempty"` +} + +// SearchHit is one result. +type SearchHit struct { + Rank float64 `json:"rank"` + Result roomserver.ClientEvent `json:"result"` + Context *SearchContext `json:"context,omitempty"` +} + +// SearchContext is what surrounds one hit. +type SearchContext struct { + Start string `json:"start,omitempty"` + End string `json:"end,omitempty"` + EventsBefore []roomserver.ClientEvent `json:"events_before"` + EventsAfter []roomserver.ClientEvent `json:"events_after"` + ProfileInfo map[string]SearchProfileInfo `json:"profile_info,omitempty"` +} + +// SearchProfileInfo is a sender's name and avatar. +type SearchProfileInfo struct { + DisplayName string `json:"displayname,omitempty"` + AvatarURL string `json:"avatar_url,omitempty"` +} + +// SearchGroupEntry is one group of a grouped search. +type SearchGroupEntry struct { + Results []string `json:"results"` + Order int `json:"order"` + NextBatch string `json:"next_batch,omitempty"` +} + +// The search tokens. A ranked search pages by offset, because the ranking +// of a result does not move as later events arrive; a search by recency +// pages by stream position, which keeps a page from repeating a result when +// new messages land between two requests. +const ( + rankTokenPrefix = "r" + recentTokenPrefix = "p" +) + +// Search answers one /search. +func (s *API) Search(ctx context.Context, req SearchRequest) (SearchResponse, error) { + criteria := req.Body.SearchCategories.RoomEvents + var resp SearchResponse + resp.SearchCategories.RoomEvents.Highlights = []string{} + resp.SearchCategories.RoomEvents.Results = []SearchHit{} + if criteria == nil { + // The only category there is was not asked for, so there is + // nothing to search. + return resp, nil + } + if strings.TrimSpace(criteria.SearchTerm) == "" { + return SearchResponse{}, errInvalidParamf("search_term must not be empty") + } + + order := storage.SearchByRank + switch criteria.OrderBy { + case "", "rank": + case "recent": + order = storage.SearchByRecent + default: + return SearchResponse{}, errInvalidParamf("order_by must be \"rank\" or \"recent\"") + } + for _, key := range criteria.Keys { + switch key { + case "content.body", "content.name", "content.topic": + default: + return SearchResponse{}, errInvalidParamf("%q is not a search key", key) + } + } + + offset, before, err := parseSearchToken(req.NextBatch, order) + if err != nil { + return SearchResponse{}, err + } + + limit := DefaultSearchLimit + var filter storage.TimelineFilter + var rooms, notRooms []string + if f := criteria.Filter; f != nil { + if f.Limit != nil && *f.Limit > 0 { + limit = min(*f.Limit, MaxSearchLimit) + } + filter = roomEventFilter(f) + rooms, notRooms = f.Rooms, f.NotRooms + } + + found, total, err := s.rooms.Search(ctx, roomserver.SearchRequest{ + User: req.User, + Term: criteria.SearchTerm, + Keys: criteria.Keys, + Rooms: rooms, + NotRooms: notRooms, + Filter: filter, + Order: order, + Offset: offset, + Before: before, + Limit: limit, + }) + if err != nil { + return SearchResponse{}, err + } + + out := &resp.SearchCategories.RoomEvents + out.Count = total + out.Highlights = storage.SearchWords(criteria.SearchTerm) + + for _, hit := range found { + h := SearchHit{Rank: hit.Rank, Result: hit.Event} + if criteria.EventContext != nil { + if h.Context, err = s.searchContext(ctx, req.User, hit, criteria); err != nil { + return SearchResponse{}, err + } + } + out.Results = append(out.Results, h) + } + + // There is more when the page came back full and the total says so. + if len(found) == limit { + switch order { + case storage.SearchByRank: + if offset+len(found) < total { + out.NextBatch = rankTokenPrefix + strconv.Itoa(offset+len(found)) + } + case storage.SearchByRecent: + out.NextBatch = recentTokenPrefix + strconv.FormatInt(found[len(found)-1].Position, 10) + } + } + + if criteria.IncludeState { + out.State = map[string][]roomserver.ClientEvent{} + for _, hit := range found { + roomID := hit.Event.RoomID + if _, done := out.State[roomID]; done { + continue + } + state, err := s.rooms.RoomState(ctx, roomID, req.User) + if err != nil { + return SearchResponse{}, err + } + out.State[roomID] = state + } + } + + if criteria.Groupings != nil { + out.Groups = searchGroups(found, criteria, out.NextBatch) + } + return resp, nil +} + +// searchContext reads the events around one hit. +func (s *API) searchContext(ctx context.Context, userID string, hit roomserver.SearchResult, criteria *RoomEventsCriteria) (*SearchContext, error) { + limitOf := func(p *int) int { + if p == nil { + return 5 + } + return max(0, min(*p, MaxSearchContext)) + } + beforeLimit := limitOf(criteria.EventContext.BeforeLimit) + afterLimit := limitOf(criteria.EventContext.AfterLimit) + roomID := hit.Event.RoomID + + out := &SearchContext{ + EventsBefore: []roomserver.ClientEvent{}, + EventsAfter: []roomserver.ClientEvent{}, + } + if beforeLimit > 0 { + earlier, err := s.rooms.Timeline(ctx, roomserver.TimelineRequest{ + RoomID: roomID, + User: userID, + High: storage.PositionBefore(hit.Position), + Direction: storage.DirectionBackward, + Limit: beforeLimit, + }) + if err != nil { + return nil, err + } + if len(earlier.Events) > 0 { + out.EventsBefore = reversed(earlier.Events) + out.Start = pduToken(earlier.Start).String() + } + } + if afterLimit > 0 { + later, err := s.rooms.Timeline(ctx, roomserver.TimelineRequest{ + RoomID: roomID, + User: userID, + Low: hit.Position, + Direction: storage.DirectionForward, + Limit: afterLimit, + }) + if err != nil { + return nil, err + } + if len(later.Events) > 0 { + out.EventsAfter = later.Events + out.End = pduToken(later.End).String() + } + } + + if criteria.EventContext.IncludeProfile { + senders := sendersOf(append(append([]roomserver.ClientEvent{hit.Event}, out.EventsBefore...), out.EventsAfter...)) + members, err := s.rooms.MemberEvents(ctx, roomID, senders) + if err != nil { + return nil, err + } + out.ProfileInfo = map[string]SearchProfileInfo{} + for _, m := range members { + if m.StateKey == nil { + continue + } + var content struct { + DisplayName string `json:"displayname"` + AvatarURL string `json:"avatar_url"` + } + _ = json.Unmarshal(m.Content, &content) + out.ProfileInfo[*m.StateKey] = SearchProfileInfo{DisplayName: content.DisplayName, AvatarURL: content.AvatarURL} + } + } + return out, nil +} + +// searchGroups groups a page of hits by room or sender, in the order the +// page ranked them. +func searchGroups(found []roomserver.SearchResult, criteria *RoomEventsCriteria, nextBatch string) map[string]map[string]SearchGroupEntry { + out := map[string]map[string]SearchGroupEntry{} + for _, g := range criteria.Groupings.GroupBy { + var keyOf func(roomserver.SearchResult) string + switch g.Key { + case "room_id": + keyOf = func(r roomserver.SearchResult) string { return r.Event.RoomID } + case "sender": + keyOf = func(r roomserver.SearchResult) string { return r.Event.Sender } + default: + continue + } + groups := map[string]SearchGroupEntry{} + for _, hit := range found { + key := keyOf(hit) + entry, ok := groups[key] + if !ok { + entry.Order = len(groups) + 1 + entry.NextBatch = nextBatch + } + entry.Results = append(entry.Results, hit.Event.EventID) + groups[key] = entry + } + out[g.Key] = groups + } + return out +} + +// parseSearchToken reads a ?next_batch=. +func parseSearchToken(raw string, order storage.SearchOrder) (offset int, before int64, err error) { + if raw == "" { + return 0, 0, nil + } + bad := errInvalidTokenf("%q is not a search token", raw) + switch order { + case storage.SearchByRecent: + body, ok := strings.CutPrefix(raw, recentTokenPrefix) + if !ok { + return 0, 0, bad + } + if before, err = strconv.ParseInt(body, 10, 64); err != nil { + return 0, 0, bad + } + return 0, before, nil + default: + body, ok := strings.CutPrefix(raw, rankTokenPrefix) + if !ok { + return 0, 0, bad + } + if offset, err = strconv.Atoi(body); err != nil || offset < 0 { + return 0, 0, fmt.Errorf("%w", bad) + } + return offset, 0, nil + } +} diff --git a/server/internal/syncapi/sliding.go b/server/internal/syncapi/sliding.go new file mode 100644 index 0000000..9a1079f --- /dev/null +++ b/server/internal/syncapi/sliding.go @@ -0,0 +1,1001 @@ +package syncapi + +import ( + "cmp" + "context" + "encoding/json" + "errors" + + "slices" + "strconv" + "strings" + "time" + + "go.opentelemetry.io/otel/attribute" + + "github.com/dariuszjaniak/warp/server/internal/matrix/event" + "github.com/dariuszjaniak/warp/server/internal/roomserver" + "github.com/dariuszjaniak/warp/server/internal/storage" + "github.com/dariuszjaniak/warp/server/internal/tracing" +) + +// Simplified Sliding Sync (MSC4186): /sync for clients with a thousand +// rooms that want to show the top twenty. +// +// A classic initial sync sends every room a user is in, with its state and +// a timeline, before the client can draw anything; a client with a large +// account waits minutes. Sliding sync turns that round. The client asks for +// lists — "my rooms, most recently active first, positions 0 to 19" — and +// for the few pieces of state it needs to draw each one, and the server +// sends those rooms and a count of the rest. As the user scrolls, the +// client widens the window; as rooms become active, they move into it and +// are sent. +// +// What the server keeps for this is per connection (slidingconn.go): which +// rooms it has sent and as of which position, so that a room is sent once +// in full and then only as it changes, and a room that leaves the window +// and comes back is not sent again unless something happened to it. The +// streams that are not about one room — to-device messages, device lists, +// account data, receipts, typing — are extensions a client turns on, and +// are read with the same helpers classic /sync uses. + +// SlidingRequest is one sliding sync request. +type SlidingRequest struct { + User string + Localpart string + Device string + + // Pos is the ?pos= of the previous response, empty for a new + // connection. + Pos string + + // Timeout is how long to wait for something to happen. + Timeout time.Duration + + // SetPresence is ?set_presence=, as for /sync. + SetPresence string + + Body SlidingBody +} + +// SlidingBody is the request body. +type SlidingBody struct { + // ConnID tells apart the connections one device keeps: a client with a + // room list and a notification process runs two. + ConnID string `json:"conn_id,omitempty"` + + Lists map[string]SlidingList `json:"lists,omitempty"` + RoomSubscriptions map[string]RoomSubscription `json:"room_subscriptions,omitempty"` + Extensions SlidingExtensionsRequest `json:"extensions,omitempty"` +} + +// RoomSubscription is what a client wants of each room it is sent. +type RoomSubscription struct { + // RequiredState lists [type, state_key] pairs. "*" is a wildcard in + // either place, "$ME" is the user, and ["m.room.member", "$LAZY"] is the + // members who sent something in the timeline. + RequiredState [][]string `json:"required_state,omitempty"` + + TimelineLimit int `json:"timeline_limit,omitempty"` +} + +// SlidingList is one room list. +type SlidingList struct { + RoomSubscription + + // Ranges are inclusive windows into the sorted list. + Ranges [][2]int `json:"ranges,omitempty"` + + Filters *SlidingFilters `json:"filters,omitempty"` +} + +// SlidingFilters narrows which rooms a list holds. +type SlidingFilters struct { + IsDM *bool `json:"is_dm,omitempty"` + Spaces []string `json:"spaces,omitempty"` + IsEncrypted *bool `json:"is_encrypted,omitempty"` + IsInvite *bool `json:"is_invite,omitempty"` + + // RoomTypes and NotRoomTypes match m.room.create's type; a null entry + // is a room with no type, which is to say an ordinary room. + RoomTypes []*string `json:"room_types,omitempty"` + NotRoomTypes []*string `json:"not_room_types,omitempty"` + + Tags []string `json:"tags,omitempty"` + NotTags []string `json:"not_tags,omitempty"` +} + +// extensionToggle is the shape every extension's request shares. +type extensionToggle struct { + Enabled *bool `json:"enabled,omitempty"` + + // Lists and Rooms narrow which rooms a per-room extension reports on; + // absent means every room in the response's lists and subscriptions. + Lists []string `json:"lists,omitempty"` + Rooms []string `json:"rooms,omitempty"` +} + +func (t *extensionToggle) on() bool { return t != nil && t.Enabled != nil && *t.Enabled } + +// SlidingExtensionsRequest is which extensions the client wants. +type SlidingExtensionsRequest struct { + ToDevice *struct { + extensionToggle + Since string `json:"since,omitempty"` + Limit int `json:"limit,omitempty"` + } `json:"to_device,omitempty"` + E2EE *extensionToggle `json:"e2ee,omitempty"` + AccountData *extensionToggle `json:"account_data,omitempty"` + Receipts *extensionToggle `json:"receipts,omitempty"` + Typing *extensionToggle `json:"typing,omitempty"` +} + +// SlidingResponse is one response. +type SlidingResponse struct { + Pos string `json:"pos"` + Lists map[string]SlidingListResult `json:"lists"` + Rooms map[string]*SlidingRoom `json:"rooms"` + Extensions SlidingExtensions `json:"extensions"` +} + +// SlidingListResult is how many rooms a list holds. +type SlidingListResult struct { + Count int `json:"count"` +} + +// SlidingHero is a member a room without a name is named after. +type SlidingHero struct { + UserID string `json:"user_id"` + DisplayName string `json:"displayname,omitempty"` + AvatarURL string `json:"avatar_url,omitempty"` +} + +// SlidingRoom is one room of a response. +type SlidingRoom struct { + Name string `json:"name,omitempty"` + Avatar string `json:"avatar,omitempty"` + Heroes []SlidingHero `json:"heroes,omitempty"` + IsDM bool `json:"is_dm,omitempty"` + + // Initial says this is the room in full: the client replaces what it + // had rather than adding to it. + Initial bool `json:"initial,omitempty"` + + RequiredState []roomserver.ClientEvent `json:"required_state,omitempty"` + Timeline []roomserver.ClientEvent `json:"timeline,omitempty"` + PrevBatch string `json:"prev_batch,omitempty"` + Limited bool `json:"limited,omitempty"` + + // NumLive is how many of the timeline's events are new since the + // previous response, rather than history sent to fill the timeline. + NumLive *int `json:"num_live,omitempty"` + + // BumpStamp orders rooms by activity: the position of the last event + // that moved the room up a list. + BumpStamp int64 `json:"bump_stamp,omitempty"` + + JoinedCount *int `json:"joined_count,omitempty"` + InvitedCount *int `json:"invited_count,omitempty"` + + NotificationCount int `json:"notification_count"` + HighlightCount int `json:"highlight_count"` + + InviteState []roomserver.StrippedEvent `json:"invite_state,omitempty"` + KnockState []roomserver.StrippedEvent `json:"knock_state,omitempty"` +} + +// SlidingExtensions is the extensions' half of a response. +type SlidingExtensions struct { + ToDevice *ToDeviceExtension `json:"to_device,omitempty"` + E2EE *E2EEExtension `json:"e2ee,omitempty"` + AccountData *AccountDataExtension `json:"account_data,omitempty"` + Receipts *EphemeralExtension `json:"receipts,omitempty"` + Typing *EphemeralExtension `json:"typing,omitempty"` +} + +// ToDeviceExtension is the to-device messages for the syncing device. +type ToDeviceExtension struct { + NextBatch string `json:"next_batch"` + Events []toDeviceEvent `json:"events"` +} + +// E2EEExtension is the end-to-end encryption bookkeeping a classic sync +// carries at its top level. +type E2EEExtension struct { + DeviceLists DeviceLists `json:"device_lists"` + DeviceOneTimeKeysCount map[string]int `json:"device_one_time_keys_count"` + DeviceUnusedFallbackKeyTypes []string `json:"device_unused_fallback_key_types"` +} + +// AccountDataExtension is global and per-room account data. +type AccountDataExtension struct { + Global []AccountDataEvent `json:"global"` + Rooms map[string][]AccountDataEvent `json:"rooms"` +} + +// EphemeralExtension is receipts or typing, by room. +type EphemeralExtension struct { + Rooms map[string]EphemeralEvent `json:"rooms"` +} + +// defaultToDeviceLimit is the to_device extension's batch size when the +// client names none. A device that has been away drains its queue over +// several requests rather than in one response of unbounded size. +const defaultToDeviceLimit = 100 + +// directType is the account data a client records its direct chats in. +const directType = "m.direct" + +// tagType is the room account data a client records a room's tags in. +const tagType = "m.tag" + +// SlidingSync answers one sliding sync request, blocking for up to +// req.Timeout when there is nothing to report. +func (s *API) SlidingSync(ctx context.Context, req SlidingRequest) (SlidingResponse, error) { + key := connKey{user: req.User, device: req.Device, connID: req.Body.ConnID} + snap, err := s.conns.load(key, req.Pos) + if err != nil { + return SlidingResponse{}, err + } + if err := s.applySetPresence(ctx, SyncRequest{User: req.User, SetPresence: req.SetPresence}); err != nil { + return SlidingResponse{}, err + } + + timeout := min(req.Timeout, s.longPollTimeout) + deadline := s.now().Add(timeout) + for { + rooms, err := s.rooms.UserMemberships(ctx, req.User, nil) + if err != nil { + return SlidingResponse{}, err + } + listener := s.notifier.Listen(req.User, roomIDsOf(rooms)) + + resp, next, interesting, err := s.slidingAttempt(ctx, req, snap, rooms) + if err != nil { + listener.Close() + return SlidingResponse{}, err + } + wait := time.Until(deadline) + if interesting || timeout <= 0 || wait <= 0 { + listener.Close() + resp.Pos = s.conns.save(key, next) + return resp, nil + } + woken, err := waitFor(ctx, listener, wait) + listener.Close() + if err != nil { + return SlidingResponse{}, err + } + if !woken { + resp.Pos = s.conns.save(key, next) + return resp, nil + } + } +} + +// slidingRoomConfig is what a room is to be sent with, merged over every +// list and subscription that selects it. +type slidingRoomConfig struct { + timelineLimit int + requiredState [][]string +} + +func (c slidingRoomConfig) merge(sub RoomSubscription) slidingRoomConfig { + c.timelineLimit = max(c.timelineLimit, min(sub.TimelineLimit, MaxTimelineLimit)) + for _, pair := range sub.RequiredState { + if len(pair) == 2 && !slices.ContainsFunc(c.requiredState, func(p []string) bool { return slices.Equal(p, pair) }) { + c.requiredState = append(c.requiredState, pair) + } + } + return c +} + +// key renders a config for comparison with the one a room was last sent +// with. +func (c slidingRoomConfig) key() string { + pairs := make([]string, 0, len(c.requiredState)) + for _, p := range c.requiredState { + pairs = append(pairs, p[0]+"\x00"+p[1]) + } + slices.Sort(pairs) + return strconv.Itoa(c.timelineLimit) + "\x01" + strings.Join(pairs, "\x01") +} + +// slidingAttempt builds one candidate response and the connection state it +// leaves behind. +func (s *API) slidingAttempt(ctx context.Context, req SlidingRequest, prev *connSnapshot, memberships []roomserver.MembershipAt) (_ SlidingResponse, _ *connSnapshot, _ bool, err error) { + ctx, span := tracing.Start(ctx, "syncapi.sliding_sync", attribute.Bool("matrix.initial", prev == nil)) + defer func() { tracing.End(span, err) }() + + positions, err := s.store.StreamPositions(ctx) + if err != nil { + return SlidingResponse{}, nil, false, err + } + to := tokenAt(positions) + + var since Token + next := &connSnapshot{token: to, startPDU: to.PDU, rooms: map[string]sentRoom{}, lists: map[string]int{}} + if prev != nil { + since = prev.token + next = prev.clone() + next.token = to + } + + resp := SlidingResponse{ + Lists: map[string]SlidingListResult{}, + Rooms: map[string]*SlidingRoom{}, + } + byRoom := make(map[string]roomserver.MembershipAt, len(memberships)) + for _, m := range memberships { + byRoom[m.RoomID] = m + } + + // The rooms a list may hold: those the user is in, invited to or + // knocking on, and those they left while this connection was open — + // which is how a client hears that a room it is showing is gone. + var candidates []roomserver.MembershipAt + for _, m := range memberships { + switch m.Membership { + case event.MembershipJoin, event.MembershipInvite, event.MembershipKnock: + candidates = append(candidates, m) + case event.MembershipLeave, event.MembershipBan: + if prev != nil && m.Position > prev.startPDU { + candidates = append(candidates, m) + } + } + } + + facts, err := s.roomFacts(ctx, req, candidates, to, req.Body.Lists) + if err != nil { + return SlidingResponse{}, nil, false, err + } + + selected := map[string]slidingRoomConfig{} + listsChanged := prev == nil + for name, list := range req.Body.Lists { + var rooms []roomserver.MembershipAt + for _, m := range candidates { + ok, err := s.listAdmits(ctx, req, list.Filters, m, facts) + if err != nil { + return SlidingResponse{}, nil, false, err + } + if ok { + rooms = append(rooms, m) + } + } + slices.SortFunc(rooms, func(a, b roomserver.MembershipAt) int { + return cmp.Or(cmp.Compare(facts.bump[b.RoomID], facts.bump[a.RoomID]), strings.Compare(a.RoomID, b.RoomID)) + }) + resp.Lists[name] = SlidingListResult{Count: len(rooms)} + if count, ok := next.lists[name]; !ok || count != len(rooms) { + listsChanged = true + } + next.lists[name] = len(rooms) + + for _, r := range list.Ranges { + lo, hi := max(r[0], 0), min(r[1], len(rooms)-1) + for i := lo; i <= hi; i++ { + selected[rooms[i].RoomID] = selected[rooms[i].RoomID].merge(list.RoomSubscription) + } + } + } + + // Subscriptions name a room directly: a room the client has open, + // whatever list it is or is not in. The user has to be able to read + // it, which for a room they are not in means a world-readable one. + for roomID, sub := range req.Body.RoomSubscriptions { + if _, ok := byRoom[roomID]; !ok { + visible, err := s.rooms.VisibleRange(ctx, roomID, req.User) + if err != nil { + if errors.Is(err, ErrNoSuchRoom) { + continue + } + return SlidingResponse{}, nil, false, err + } + if !visible.Allowed { + continue + } + byRoom[roomID] = roomserver.MembershipAt{RoomID: roomID} + } + selected[roomID] = selected[roomID].merge(sub) + } + + changed, err := s.slidingChangedRooms(ctx, req, prev, selected, byRoom, since, to) + if err != nil { + return SlidingResponse{}, nil, false, err + } + + selectedIDs := make([]string, 0, len(selected)) + for roomID := range selected { + selectedIDs = append(selectedIDs, roomID) + } + slices.Sort(selectedIDs) + + var counts map[string]storage.NotificationCounts + if s.push != nil && len(changed) > 0 { + if counts, err = s.push.NotificationCounts(ctx, req.User, selectedIDs); err != nil { + return SlidingResponse{}, nil, false, err + } + } + + initialRooms := map[string]bool{} + for _, roomID := range selectedIDs { + initial, ok := changed[roomID] + if !ok { + continue + } + config := selected[roomID] + m := byRoom[roomID] + sent := next.rooms[roomID] + from := sent.pos + if initial { + from = 0 + } + room, err := s.slidingRoom(ctx, req, m, config, from, to, facts) + if err != nil { + if errors.Is(err, ErrForbidden) { + // Lost access between the membership read and this one. + continue + } + return SlidingResponse{}, nil, false, err + } + room.Initial = initial + room.BumpStamp = facts.bump[roomID] + c := counts[roomID] + room.NotificationCount, room.HighlightCount = c.Notifications, c.Highlights + resp.Rooms[roomID] = room + if initial { + initialRooms[roomID] = true + } + next.rooms[roomID] = sentRoom{pos: to.PDU, config: config.key(), membership: m.Membership, typing: sent.typing} + } + + extensionsInteresting, err := s.slidingExtensions(ctx, req, prev, next, since, to, memberships, selectedIDs, byRoom, initialRooms, &resp) + if err != nil { + return SlidingResponse{}, nil, false, err + } + + interesting := prev == nil || listsChanged || len(resp.Rooms) > 0 || extensionsInteresting + return resp, next, interesting, nil +} + +// slidingFacts are what a room list is filtered and sorted by. +type slidingFacts struct { + bump map[string]int64 + dms map[string]bool + entry map[string]roomserver.RoomListEntry +} + +// roomFacts reads what the lists need to know about the candidate rooms. +// +// The bump stamp of a joined room is its last activity; a room the user is +// invited to, knocking on or has left is bumped by their own membership, +// because they cannot see what happens in it. +func (s *API) roomFacts(ctx context.Context, req SlidingRequest, candidates []roomserver.MembershipAt, to Token, lists map[string]SlidingList) (slidingFacts, error) { + facts := slidingFacts{bump: map[string]int64{}, dms: map[string]bool{}, entry: map[string]roomserver.RoomListEntry{}} + + var joined []string + for _, m := range candidates { + if m.Membership == event.MembershipJoin { + joined = append(joined, m.RoomID) + } else { + facts.bump[m.RoomID] = m.Position + } + } + bumps, err := s.rooms.BumpStamps(ctx, joined, to.PDU) + if err != nil { + return slidingFacts{}, err + } + for _, roomID := range joined { + facts.bump[roomID] = bumps[roomID] + } + + raw, err := s.accounts.AccountData(ctx, req.Localpart, "", directType) + switch { + case errors.Is(err, storage.ErrNoSuchAccountData): + case err != nil: + return slidingFacts{}, err + default: + var direct map[string][]string + if json.Unmarshal(raw, &direct) == nil { + for _, rooms := range direct { + for _, roomID := range rooms { + facts.dms[roomID] = true + } + } + } + } + return facts, nil +} + +// entryOf reads a room's list entry once per attempt. +func (s *API) entryOf(ctx context.Context, roomID string, facts slidingFacts) (roomserver.RoomListEntry, error) { + if e, ok := facts.entry[roomID]; ok { + return e, nil + } + e, err := s.rooms.RoomListEntry(ctx, roomID) + if err != nil { + return roomserver.RoomListEntry{}, err + } + facts.entry[roomID] = e + return e, nil +} + +// listAdmits applies a list's filters to one room. +func (s *API) listAdmits(ctx context.Context, req SlidingRequest, f *SlidingFilters, m roomserver.MembershipAt, facts slidingFacts) (bool, error) { + if f == nil { + return true, nil + } + if f.IsDM != nil && facts.dms[m.RoomID] != *f.IsDM { + return false, nil + } + if f.IsInvite != nil && (m.Membership == event.MembershipInvite) != *f.IsInvite { + return false, nil + } + if f.IsEncrypted != nil || len(f.RoomTypes) > 0 || len(f.NotRoomTypes) > 0 { + entry, err := s.entryOf(ctx, m.RoomID, facts) + if err != nil { + return false, err + } + if f.IsEncrypted != nil && entry.Encrypted != *f.IsEncrypted { + return false, nil + } + typeMatches := func(types []*string) bool { + return slices.ContainsFunc(types, func(t *string) bool { + if t == nil { + return entry.RoomType == "" + } + return *t == entry.RoomType + }) + } + if len(f.RoomTypes) > 0 && !typeMatches(f.RoomTypes) { + return false, nil + } + if typeMatches(f.NotRoomTypes) { + return false, nil + } + } + if len(f.Spaces) > 0 { + in := false + for _, space := range f.Spaces { + children, err := s.rooms.SpaceChildRooms(ctx, space) + if err != nil { + return false, err + } + if slices.Contains(children, m.RoomID) { + in = true + break + } + } + if !in { + return false, nil + } + } + if len(f.Tags) > 0 || len(f.NotTags) > 0 { + tags, err := s.roomTags(ctx, req.Localpart, m.RoomID) + if err != nil { + return false, err + } + if len(f.Tags) > 0 && !slices.ContainsFunc(f.Tags, func(t string) bool { return tags[t] }) { + return false, nil + } + if slices.ContainsFunc(f.NotTags, func(t string) bool { return tags[t] }) { + return false, nil + } + } + return true, nil +} + +// roomTags reads the tags a user has put on a room. +func (s *API) roomTags(ctx context.Context, localpart, roomID string) (map[string]bool, error) { + raw, err := s.accounts.AccountData(ctx, localpart, roomID, tagType) + if errors.Is(err, storage.ErrNoSuchAccountData) { + return nil, nil + } + if err != nil { + return nil, err + } + var content struct { + Tags map[string]json.RawMessage `json:"tags"` + } + // Tags a client wrote in a shape that is not the spec's are no tags: + // a list filter is not the place to refuse somebody's account data. + if json.Unmarshal(raw, &content) != nil { + content.Tags = nil + } + out := make(map[string]bool, len(content.Tags)) + for t := range content.Tags { + out[t] = true + } + return out, nil +} + +// slidingChangedRooms decides which selected rooms go into the response, +// and whether each goes in full. The value is true for a room sent in full. +func (s *API) slidingChangedRooms(ctx context.Context, req SlidingRequest, prev *connSnapshot, selected map[string]slidingRoomConfig, byRoom map[string]roomserver.MembershipAt, since, to Token) (map[string]bool, error) { + out := map[string]bool{} + // The rooms already sent, grouped by the position they were sent up + // to, so that "has anything happened since" is one query per group. + byPos := map[int64][]string{} + for roomID, config := range selected { + var sent sentRoom + var ok bool + if prev != nil { + sent, ok = prev.rooms[roomID] + } + m := byRoom[roomID] + switch { + case !ok, sent.config != config.key(): + out[roomID] = true + case m.Membership != sent.membership: + // A membership the room was last sent under no longer + // applies: an invite accepted is a room the client has only + // seen stripped state of. + out[roomID] = sent.membership == event.MembershipInvite || + sent.membership == event.MembershipKnock || sent.membership == "" + case m.Position > sent.pos: + out[roomID] = false + case m.Membership == event.MembershipJoin || m.Membership == "": + byPos[sent.pos] = append(byPos[sent.pos], roomID) + } + } + for pos, roomIDs := range byPos { + slices.Sort(roomIDs) + withEvents, err := s.rooms.RoomsWithEvents(ctx, roomIDs, pos, to.PDU) + if err != nil { + return nil, err + } + for _, roomID := range withEvents { + out[roomID] = false + } + } + + // Rooms whose unread counts moved without an event: read on another + // device. + if s.push != nil && prev != nil && to.Notification > since.Notification { + moved, err := s.push.NotificationCountChanges(ctx, req.User, since.Notification, to.Notification) + if err != nil { + return nil, err + } + for _, roomID := range moved { + if _, ok := selected[roomID]; ok { + if _, already := out[roomID]; !already { + out[roomID] = false + } + } + } + } + return out, nil +} + +// slidingRoom reads one room for a response. from is where the client's +// copy of the room ends, zero for a room sent in full. +func (s *API) slidingRoom(ctx context.Context, req SlidingRequest, m roomserver.MembershipAt, config slidingRoomConfig, from int64, to Token, facts slidingFacts) (*SlidingRoom, error) { + room := &SlidingRoom{IsDM: facts.dms[m.RoomID]} + entry, err := s.entryOf(ctx, m.RoomID, facts) + if err != nil { + return nil, err + } + room.Name, room.Avatar = entry.Name, entry.AvatarURL + + switch m.Membership { + case event.MembershipInvite, event.MembershipKnock: + state, err := s.rooms.StrippedState(ctx, m.RoomID, []string{req.User}) + if err != nil { + return nil, err + } + if m.Membership == event.MembershipInvite { + room.InviteState = state + } else { + room.KnockState = state + } + return room, nil + } + + if config.timelineLimit > 0 { + timeline, err := s.rooms.Timeline(ctx, roomserver.TimelineRequest{ + RoomID: m.RoomID, + User: req.User, + Low: from, + High: to.PDU, + Direction: storage.DirectionBackward, + Limit: config.timelineLimit, + }) + if err != nil { + return nil, err + } + room.Timeline = withoutRoomID(timeline.Events) + room.Limited = timeline.Limited + if len(timeline.Events) > 0 { + room.PrevBatch = pduToken(timeline.Start).String() + } + live := 0 + if from != 0 { + live = len(timeline.Events) + } + room.NumLive = &live + } + + match := newStateMatcher(config.requiredState, req.User, room.Timeline) + if !match.empty() { + state, err := s.rooms.RequiredState(ctx, roomserver.RequiredStateRequest{ + RoomID: m.RoomID, + User: req.User, + Since: from, + To: to.PDU, + Match: match.matches, + }) + if err != nil { + return nil, err + } + room.RequiredState = withoutRoomID(state) + } + + summary, err := s.rooms.RoomSummaryFor(ctx, m.RoomID, req.User) + if err != nil { + return nil, err + } + joined, invited := summary.JoinedMemberCount, summary.InvitedMemberCount + room.JoinedCount, room.InvitedCount = &joined, &invited + if room.Name == "" && len(summary.Heroes) > 0 { + members, err := s.rooms.MemberEvents(ctx, m.RoomID, summary.Heroes) + if err != nil { + return nil, err + } + profiles := map[string]SlidingHero{} + for _, e := range members { + if e.StateKey == nil { + continue + } + var content struct { + DisplayName string `json:"displayname"` + AvatarURL string `json:"avatar_url"` + } + _ = json.Unmarshal(e.Content, &content) + profiles[*e.StateKey] = SlidingHero{UserID: *e.StateKey, DisplayName: content.DisplayName, AvatarURL: content.AvatarURL} + } + for _, userID := range summary.Heroes { + hero, ok := profiles[userID] + if !ok { + hero = SlidingHero{UserID: userID} + } + room.Heroes = append(room.Heroes, hero) + } + } + return room, nil +} + +// stateMatcher is a room's required_state, compiled. +type stateMatcher struct { + all bool + wholeType map[string]bool + anyType map[string]bool + tuples map[[2]string]bool +} + +func newStateMatcher(required [][]string, userID string, timeline []roomserver.ClientEvent) stateMatcher { + m := stateMatcher{wholeType: map[string]bool{}, anyType: map[string]bool{}, tuples: map[[2]string]bool{}} + for _, pair := range required { + eventType, stateKey := pair[0], pair[1] + switch stateKey { + case "$ME": + stateKey = userID + case "$LAZY": + if eventType == event.TypeMember { + // The members whose names the timeline needs, and the user + // themselves, whose own membership a client always shows. + m.tuples[[2]string{event.TypeMember, userID}] = true + for _, e := range timeline { + m.tuples[[2]string{event.TypeMember, e.Sender}] = true + } + } + continue + } + switch { + case eventType == "*" && stateKey == "*": + m.all = true + case stateKey == "*": + m.wholeType[eventType] = true + case eventType == "*": + m.anyType[stateKey] = true + default: + m.tuples[[2]string{eventType, stateKey}] = true + } + } + return m +} + +func (m stateMatcher) empty() bool { + return !m.all && len(m.wholeType) == 0 && len(m.anyType) == 0 && len(m.tuples) == 0 +} + +func (m stateMatcher) matches(eventType, stateKey string) bool { + return m.all || m.wholeType[eventType] || m.anyType[stateKey] || m.tuples[[2]string{eventType, stateKey}] +} + +// extensionRooms narrows the rooms a per-room extension reports on to the +// ones its request names, by list or by room. +// +// Naming any list the request has, or "*", is every selected room: the +// windows of the lists are merged before the extensions run, and a client +// naming its main list means the rooms it is showing, which is a superset +// at worst. Naming rooms narrows to those of them that are selected. +func extensionRooms(t *extensionToggle, body SlidingBody, selected []string) []string { + if len(t.Lists) == 0 && len(t.Rooms) == 0 { + return selected + } + for _, name := range t.Lists { + if _, ok := body.Lists[name]; ok || name == "*" { + return selected + } + } + if slices.Contains(t.Rooms, "*") { + return selected + } + var out []string + for _, roomID := range t.Rooms { + if slices.Contains(selected, roomID) { + out = append(out, roomID) + } + } + return out +} + +// slidingExtensions fills in the extensions a request turned on, and +// reports whether any of them has something to say. +func (s *API) slidingExtensions( + ctx context.Context, + req SlidingRequest, + prev, next *connSnapshot, + since, to Token, + memberships []roomserver.MembershipAt, + selected []string, + byRoom map[string]roomserver.MembershipAt, + initialRooms map[string]bool, + resp *SlidingResponse, +) (bool, error) { + ext := req.Body.Extensions + interesting := false + + if ext.ToDevice != nil && ext.ToDevice.on() { + from := next.toDevice + if ext.ToDevice.Since != "" { + parsed, err := strconv.ParseInt(ext.ToDevice.Since, 10, 64) + if err != nil || parsed < 0 { + return false, errInvalidTokenf("%q is not a to_device token", ext.ToDevice.Since) + } + from = parsed + // Coming back with a since acknowledges everything up to it. + s.ackToDevice(ctx, req.Localpart, req.Device, from) + } + limit := ext.ToDevice.Limit + if limit <= 0 { + limit = defaultToDeviceLimit + } + msgs, pos, err := s.toDeviceMessages(ctx, req.Localpart, req.Device, from, to.ToDevice, limit) + if err != nil { + return false, err + } + if msgs == nil { + msgs = []toDeviceEvent{} + } + next.toDevice = pos + resp.Extensions.ToDevice = &ToDeviceExtension{NextBatch: strconv.FormatInt(pos, 10), Events: msgs} + interesting = interesting || len(msgs) > 0 + } + + if ext.E2EE.on() { + e2ee := &E2EEExtension{DeviceLists: DeviceLists{Changed: []string{}, Left: []string{}}} + if prev != nil { + lists, err := s.deviceLists(ctx, req.User, since, to, memberships) + if err != nil { + return false, err + } + e2ee.DeviceLists = lists + } + var keys Response + if err := s.deviceKeyCounts(ctx, SyncRequest{User: req.User, Device: req.Device}, &keys); err != nil { + return false, err + } + e2ee.DeviceOneTimeKeysCount = keys.DeviceOneTimeKeysCount + if e2ee.DeviceOneTimeKeysCount == nil { + e2ee.DeviceOneTimeKeysCount = map[string]int{} + } + e2ee.DeviceUnusedFallbackKeyTypes = []string{} + if keys.DeviceUnusedFallbackKeyTypes != nil { + e2ee.DeviceUnusedFallbackKeyTypes = *keys.DeviceUnusedFallbackKeyTypes + } + resp.Extensions.E2EE = e2ee + interesting = interesting || len(e2ee.DeviceLists.Changed) > 0 || len(e2ee.DeviceLists.Left) > 0 + } + + if ext.AccountData.on() { + from := since.AccountData + global, err := s.globalAccountData(ctx, SyncRequest{User: req.User, Localpart: req.Localpart}, from, to.AccountData, prev == nil, Filter{}) + if err != nil { + return false, err + } + out := &AccountDataExtension{Global: global, Rooms: map[string][]AccountDataEvent{}} + for _, roomID := range extensionRooms(ext.AccountData, req.Body, selected) { + window := roomWindow{since: since, to: to, full: initialRooms[roomID] || prev == nil} + events, err := s.roomAccountData(ctx, req.Localpart, roomID, window, Filter{}) + if err != nil { + return false, err + } + if len(events) > 0 { + out.Rooms[roomID] = events + } + } + resp.Extensions.AccountData = out + interesting = interesting || len(global) > 0 || len(out.Rooms) > 0 + } + + joinedOnly := func(roomIDs []string) []string { + var out []string + for _, roomID := range roomIDs { + if byRoom[roomID].Membership == event.MembershipJoin { + out = append(out, roomID) + } + } + return out + } + + if ext.Receipts.on() { + out := &EphemeralExtension{Rooms: map[string]EphemeralEvent{}} + rooms := joinedOnly(extensionRooms(ext.Receipts, req.Body, selected)) + var fresh, ongoing []string + for _, roomID := range rooms { + if initialRooms[roomID] || prev == nil { + fresh = append(fresh, roomID) + } else { + ongoing = append(ongoing, roomID) + } + } + for _, set := range []struct { + rooms []string + from int64 + }{{fresh, 0}, {ongoing, since.Receipt}} { + if len(set.rooms) == 0 || (set.from != 0 && to.Receipt <= set.from) { + continue + } + byRoomReceipts, err := s.roomReceipts(ctx, req.User, set.rooms, set.from, to.Receipt) + if err != nil { + return false, err + } + for roomID, receipts := range byRoomReceipts { + if e := receiptsEvent(receipts); e != nil { + out.Rooms[roomID] = *e + } + } + } + resp.Extensions.Receipts = out + interesting = interesting || len(out.Rooms) > 0 + } + + if ext.Typing.on() { + out := &EphemeralExtension{Rooms: map[string]EphemeralEvent{}} + for _, roomID := range joinedOnly(extensionRooms(ext.Typing, req.Body, selected)) { + e := s.typingEvent(roomID) + current := "" + if e != nil { + current = string(e.Content) + } + sent := next.rooms[roomID] + if current == sent.typing { + continue + } + if e == nil { + // Somebody stopped: the client has to be told the list is + // now empty, or it would show them typing until it + // restarted. + e = &EphemeralEvent{Type: "m.typing", Content: json.RawMessage(`{"user_ids":[]}`)} + } + out.Rooms[roomID] = *e + sent.typing = current + next.rooms[roomID] = sent + } + resp.Extensions.Typing = out + interesting = interesting || len(out.Rooms) > 0 + } + return interesting, nil +} diff --git a/server/internal/syncapi/slidingconn.go b/server/internal/syncapi/slidingconn.go new file mode 100644 index 0000000..e71368b --- /dev/null +++ b/server/internal/syncapi/slidingconn.go @@ -0,0 +1,219 @@ +package syncapi + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "maps" + "slices" + "strconv" + "strings" + "sync" + "time" +) + +// Sliding sync connections: what the server remembers about what each +// client has already been sent. +// +// A classic sync token is a position in every stream and nothing else; the +// client is sent everything after it. Sliding sync cannot work that way, +// because what a client is sent depends on what it asked for — which rooms +// are in the window of which list, with which pieces of state — and that +// changes from request to request. So the server remembers, per connection, +// which rooms it has sent and as of which position, and `pos` names one of +// those memories. +// +// The memory is in process and is lost on restart. That is what the +// protocol expects: an unknown `pos` is answered with M_UNKNOWN_POS, and +// the client starts a new connection — one initial request, after which it +// is in step again. Persisting it would buy a slightly cheaper restart for +// a write on every request. + +// ErrUnknownPos is a `pos` this server does not remember: expired, from +// before a restart, or never handed out. +var ErrUnknownPos = errors.New("unknown sliding sync position") + +// Limits of the connection store. +const ( + // connectionTTL is how long an idle connection is remembered. A client + // that comes back later starts again. + connectionTTL = 30 * time.Minute + + // maxConnectionsPerDevice caps the connections one device may hold + // open, each conn_id being one. The oldest goes first. + maxConnectionsPerDevice = 10 + + // keptGenerations is how many of a connection's latest positions stay + // usable. A client that never saw a response retries with the pos it + // sent, so the one before the latest has to still work. + keptGenerations = 3 +) + +// sentRoom is what a connection has been sent of one room. +type sentRoom struct { + // pos is the event stream position the room was last sent up to. + pos int64 + + // config is the room's required_state and timeline_limit as of that + // send, rendered as a string: when it changes, the state the client + // holds is no longer what it asked for. + config string + + // membership is the user's membership as of that send. + membership string + + // typing is the typing notification last sent, as its JSON. + typing string +} + +// connSnapshot is a connection as of one `pos`. +type connSnapshot struct { + // token is the stream positions the response at this pos read up to. + token Token + + // startPDU is the event stream position the connection began at: rooms + // the user left before it are not news to it. + startPDU int64 + + rooms map[string]sentRoom + + // lists is each list's room count as last sent. + lists map[string]int + + // toDevice is where the to_device extension has got to. + toDevice int64 +} + +// clone copies a snapshot so that a new generation can be built from it +// without touching the one a retry might still need. +func (c *connSnapshot) clone() *connSnapshot { + return &connSnapshot{ + token: c.token, + startPDU: c.startPDU, + rooms: maps.Clone(c.rooms), + lists: maps.Clone(c.lists), + toDevice: c.toDevice, + } +} + +// slidingConn is one connection: a client device's conn_id. +type slidingConn struct { + // nonce makes the connection's positions its own: a pos from another + // connection, or from before a restart, cannot name one of these + // generations by accident. + nonce string + + generations map[int64]*connSnapshot + latest int64 + lastUsed time.Time +} + +type connKey struct { + user, device, connID string +} + +// connStore holds every connection. +type connStore struct { + mu sync.Mutex + conns map[connKey]*slidingConn + now func() time.Time +} + +func newConnStore(now func() time.Time) *connStore { + return &connStore{conns: map[connKey]*slidingConn{}, now: now} +} + +// load returns the snapshot a request's pos names, or a fresh one for a +// request with none. A request without a pos starts the connection over, +// which is what a client does after M_UNKNOWN_POS and on first launch. +func (s *connStore) load(key connKey, pos string) (*connSnapshot, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.sweep() + + if pos == "" { + delete(s.conns, key) + return nil, nil + } + conn, ok := s.conns[key] + if !ok { + return nil, fmt.Errorf("%w: the connection has expired", ErrUnknownPos) + } + nonce, raw, ok := strings.Cut(pos, ".") + gen, err := strconv.ParseInt(raw, 10, 64) + if !ok || err != nil || nonce != conn.nonce { + return nil, fmt.Errorf("%w: %q is not a position of this connection", ErrUnknownPos, pos) + } + snap, ok := conn.generations[gen] + if !ok { + return nil, fmt.Errorf("%w: %q is too old", ErrUnknownPos, pos) + } + conn.lastUsed = s.now() + return snap, nil +} + +// save stores a new generation of a connection and returns its pos. +func (s *connStore) save(key connKey, snap *connSnapshot) string { + s.mu.Lock() + defer s.mu.Unlock() + + conn, ok := s.conns[key] + if !ok { + conn = &slidingConn{nonce: randomNonce(), generations: map[int64]*connSnapshot{}, lastUsed: s.now()} + s.conns[key] = conn + s.capDevice(key) + } + conn.latest++ + conn.generations[conn.latest] = snap + delete(conn.generations, conn.latest-keptGenerations) + conn.lastUsed = s.now() + return conn.nonce + "." + strconv.FormatInt(conn.latest, 10) +} + +// sweep forgets idle connections. It runs under the lock on every load, +// which keeps the store bounded without a goroutine of its own. +func (s *connStore) sweep() { + cutoff := s.now().Add(-connectionTTL) + for key, conn := range s.conns { + if conn.lastUsed.Before(cutoff) { + delete(s.conns, key) + } + } +} + +// capDevice drops a device's oldest connections beyond the cap. The +// connection being saved is never one of them. +func (s *connStore) capDevice(key connKey) { + var others []connKey + for k := range s.conns { + if k != key && k.user == key.user && k.device == key.device { + others = append(others, k) + } + } + slices.SortFunc(others, func(a, b connKey) int { + return s.conns[a].lastUsed.Compare(s.conns[b].lastUsed) + }) + for len(others) > maxConnectionsPerDevice-1 { + delete(s.conns, others[0]) + others = others[1:] + } +} + +// connections reports how many connections are held, for tests and +// metrics. +func (s *connStore) connections() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.conns) +} + +func randomNonce() string { + var b [6]byte + _, _ = rand.Read(b[:]) + return hex.EncodeToString(b[:]) +} + +// SlidingConnections reports how many sliding sync connections the server +// is holding state for. +func (s *API) SlidingConnections() int { return s.conns.connections() } diff --git a/server/internal/syncapi/sync.go b/server/internal/syncapi/sync.go index 010e45b..12cfbb4 100644 --- a/server/internal/syncapi/sync.go +++ b/server/internal/syncapi/sync.go @@ -7,9 +7,12 @@ import ( "slices" "time" + "go.opentelemetry.io/otel/attribute" + "github.com/dariuszjaniak/warp/server/internal/matrix/event" "github.com/dariuszjaniak/warp/server/internal/roomserver" "github.com/dariuszjaniak/warp/server/internal/storage" + "github.com/dariuszjaniak/warp/server/internal/tracing" ) // GET /sync: the endpoint a client spends its life in. @@ -103,6 +106,12 @@ type JoinedRoom struct { // it out rather than claiming zero: an absent count says nothing, // where a zero says "you have read everything". UnreadNotifications *UnreadNotifications `json:"unread_notifications,omitempty"` + + // UnreadThreadNotifications is the counts of each thread with + // something unread, keyed by the thread root's event ID, for a filter + // that asks for them; UnreadNotifications then counts the main + // timeline alone. + UnreadThreadNotifications map[string]UnreadNotifications `json:"unread_thread_notifications,omitempty"` } // InvitedRoom is a room the user has been invited to. It carries no @@ -266,7 +275,10 @@ func (s *API) Sync(ctx context.Context, req SyncRequest) (Response, error) { // blocking on a change it has just missed. The listener is returned rather // than closed here because the caller is the one that knows whether it is // about to wait on it. -func (s *API) attempt(ctx context.Context, req SyncRequest, filter Filter, since Token) (Response, bool, *Listener, error) { +func (s *API) attempt(ctx context.Context, req SyncRequest, filter Filter, since Token) (_ Response, _ bool, _ *Listener, err error) { + ctx, span := tracing.Start(ctx, "syncapi.sync", attribute.Bool("matrix.initial", since.IsInitial())) + defer func() { tracing.End(span, err) }() + rooms, err := s.rooms.UserMemberships(ctx, req.User, nil) if err != nil { return Response{}, false, nil, err @@ -333,17 +345,18 @@ func (s *API) build(ctx context.Context, req SyncRequest, filter Filter, since T if err := s.buildRooms(ctx, req, filter, since, to, rooms, &resp); err != nil { return Response{}, false, err } - if err := s.unreadNotifications(ctx, req.User, &resp); err != nil { + if err := s.unreadNotifications(ctx, req.User, filter.UnreadThreadNotifications(), &resp); err != nil { return Response{}, false, err } - accountData, err := s.globalAccountData(ctx, req, since.AccountData, to.AccountData, filter) + accountData, err := s.globalAccountData(ctx, req, since.AccountData, to.AccountData, + req.Since == "" || req.FullState, filter) if err != nil { return Response{}, false, err } resp.AccountData.Events = accountData - toDevice, toDevicePos, err := s.toDeviceMessages(ctx, req.Localpart, req.Device, since.ToDevice, to.ToDevice) + toDevice, toDevicePos, err := s.toDeviceMessages(ctx, req.Localpart, req.Device, since.ToDevice, to.ToDevice, 0) if err != nil { return Response{}, false, err } @@ -373,7 +386,7 @@ func (s *API) build(ctx context.Context, req SyncRequest, filter Filter, since T } resp.NextBatch = next.String() - return resp, isInteresting(resp, since), nil + return resp, isInteresting(resp, req.Since == ""), nil } // isInteresting reports whether a response is worth returning rather than @@ -385,8 +398,13 @@ func (s *API) build(ctx context.Context, req SyncRequest, filter Filter, since T // into a busy loop. So the test is whether the response has anything in it, // with an initial sync as the exception: a client that has just logged in // needs an answer whatever is in it. -func isInteresting(resp Response, since Token) bool { - if since.IsInitial() { +// +// Initial means the client sent no ?since=, not that the token it sent is +// all zeroes. On a server whose streams have not moved yet — no events, no +// receipts, nothing — "s0" is a client's every token, and a sync that took +// that for a first one would never wait. +func isInteresting(resp Response, initial bool) bool { + if initial { return true } return len(resp.Rooms.Join) > 0 || @@ -665,7 +683,8 @@ func (s *API) roomEphemeral(ctx context.Context, userID, roomID string, window r } // globalAccountData reads the account data that is not about a room. -func (s *API) globalAccountData(ctx context.Context, req SyncRequest, from, to int64, filter Filter) ([]AccountDataEvent, error) { +// initial is whether this is the client's first read of it. +func (s *API) globalAccountData(ctx context.Context, req SyncRequest, from, to int64, initial bool, filter Filter) ([]AccountDataEvent, error) { entries, err := s.accountDataEntries(ctx, req.Localpart, from, to) if err != nil { return nil, err @@ -673,7 +692,13 @@ func (s *API) globalAccountData(ctx context.Context, req SyncRequest, from, to i // Every user has push rules, written or not: a user who has never // changed one has the server defaults, and a client reads its rules // from the account data of its first sync whichever it is. - if from == 0 && s.push != nil && !slices.ContainsFunc(entries, func(e storage.AccountDataEntry) bool { + // + // "First" is the client's first sync, not a token whose account data + // position is zero: on a server where nobody has written any account + // data yet every token has a zero there, and treating each sync as the + // first would send the rules every time — which makes every sync + // "interesting", and turns every long poll into a busy loop. + if initial && s.push != nil && !slices.ContainsFunc(entries, func(e storage.AccountDataEntry) bool { return e.RoomID == "" && e.DataType == pushRulesType }) { entries = append(entries, storage.AccountDataEntry{DataType: pushRulesType}) diff --git a/server/internal/syncapi/syncapi.go b/server/internal/syncapi/syncapi.go index f6f1445..4b9d3b7 100644 --- a/server/internal/syncapi/syncapi.go +++ b/server/internal/syncapi/syncapi.go @@ -84,12 +84,14 @@ type Keys interface { } // Push is the part of the push server a sync reads: the push rules it -// reports as account data, the unread counts of each room, and which +// reports as account data, the unread counts of each room (and of each +// thread in it, for a client that asks), and which // rooms' counts moved. It is also told about read receipts, which are what // clear the counts. *pushserver.API satisfies it. type Push interface { Document(ctx context.Context, localpart, userID string) ([]byte, error) NotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]storage.NotificationCounts, error) + ThreadNotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]map[string]storage.NotificationCounts, error) NotificationCountChanges(ctx context.Context, userID string, from, to int64) ([]string, error) ReceiptSet(ctx context.Context, userID, roomID, receiptType, threadID, eventID string) } @@ -104,8 +106,11 @@ type API struct { store storage.SyncStore notifier *Notifier typing *typingTracker - log *slog.Logger - now func() time.Time + + // conns remembers what each sliding sync connection has been sent. + conns *connStore + log *slog.Logger + now func() time.Time // presenceEnabled is the config switch of PLAN.md open question 3. // With presence off the endpoints answer M_FORBIDDEN and the presence @@ -227,6 +232,7 @@ func New(opts Options) (*API, error) { store: opts.Store, notifier: opts.Notifier, typing: newTypingTracker(opts.TypingTimeout, opts.Notifier, opts.Now), + conns: newConnStore(opts.Now), log: opts.Logger, now: opts.Now, presenceEnabled: opts.PresenceEnabled, diff --git a/server/internal/syncapi/todevice.go b/server/internal/syncapi/todevice.go index 62f707d..1cd74cd 100644 --- a/server/internal/syncapi/todevice.go +++ b/server/internal/syncapi/todevice.go @@ -140,14 +140,14 @@ type toDeviceEvent struct { // position of the last message *delivered* rather than the top of the // stream: a queue longer than one batch is drained over several syncs, and // a token naming the top would skip the rest. -func (s *API) toDeviceMessages(ctx context.Context, localpart, deviceID string, from, to int64) ([]toDeviceEvent, int64, error) { +func (s *API) toDeviceMessages(ctx context.Context, localpart, deviceID string, from, to int64, limit int) ([]toDeviceEvent, int64, error) { if deviceID == "" { // An access token always names a device, so this cannot happen // through the API; a guest session later might. return nil, from, nil } - msgs, err := s.store.ToDeviceMessages(ctx, localpart, deviceID, from, to, 0) + msgs, err := s.store.ToDeviceMessages(ctx, localpart, deviceID, from, to, limit) if err != nil { return nil, 0, err } diff --git a/server/internal/tracing/tracing.go b/server/internal/tracing/tracing.go new file mode 100644 index 0000000..6b1bb72 --- /dev/null +++ b/server/internal/tracing/tracing.go @@ -0,0 +1,199 @@ +// Package tracing is Warp's OpenTelemetry tracing: a span per HTTP +// request, spans around the work that request does in the components +// (persisting an event, building a sync, calling another server), and the +// W3C trace context passed on to the servers and services Warp calls, so +// that a federation request or an application service transaction shows up +// in the same trace as the client request that caused it. +// +// Tracing is off unless tracing.enabled is set. Off, the global tracer +// provider is OpenTelemetry's no-op one, and every span below costs a +// function call that returns at once — which is what lets the components +// start spans unconditionally rather than checking a flag. +// +// Spans are exported over OTLP/HTTP, which every collector and every +// tracing backend worth running (Jaeger, Tempo, Honeycomb, an OpenTelemetry +// Collector in front of anything else) accepts. +package tracing + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.43.0" + "go.opentelemetry.io/otel/trace" + + "github.com/dariuszjaniak/warp/server/internal/httputil" +) + +// instrumentation is the name Warp's spans are reported under. +const instrumentation = "github.com/dariuszjaniak/warp/server" + +// Config is what Setup needs. +type Config struct { + Enabled bool + + // Endpoint is the OTLP/HTTP collector, e.g. "http://localhost:4318". + // Empty leaves it to the OTEL_EXPORTER_OTLP_* environment variables, + // and to the exporter's default of localhost:4318 without them. + Endpoint string + + // SampleRatio is the fraction of traces started here that are kept, + // from 0 to 1. A request that arrives with a sampled parent is always + // kept, whatever the ratio: the caller has already decided. + SampleRatio float64 + + ServiceName string + ServiceVersion string + + // ServerName is the Matrix server name, recorded on every span so that + // traces from several homeservers sharing a collector can be told + // apart. + ServerName string +} + +// Setup installs the global tracer provider and propagator, and returns +// the function that flushes and stops them. +func Setup(ctx context.Context, cfg Config) (func(context.Context) error, error) { + // The propagator is installed even with tracing off: a Warp that does + // not record still passes on the trace context it was given, so that a + // trace through it is not broken in two. + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, propagation.Baggage{})) + if !cfg.Enabled { + return func(context.Context) error { return nil }, nil + } + + var opts []otlptracehttp.Option + if cfg.Endpoint != "" { + u, err := url.Parse(cfg.Endpoint) + if err != nil || u.Host == "" { + return nil, fmt.Errorf("tracing: endpoint %q is not a URL", cfg.Endpoint) + } + opts = append(opts, otlptracehttp.WithEndpointURL(cfg.Endpoint)) + } + exporter, err := otlptracehttp.New(ctx, opts...) + if err != nil { + return nil, fmt.Errorf("tracing: build the OTLP exporter: %w", err) + } + + name := cfg.ServiceName + if name == "" { + name = "warp" + } + res, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL, + semconv.ServiceName(name), + semconv.ServiceVersion(cfg.ServiceVersion), + attribute.String("matrix.server_name", cfg.ServerName), + )) + if err != nil && !errors.Is(err, resource.ErrSchemaURLConflict) { + return nil, fmt.Errorf("tracing: describe the service: %w", err) + } + + provider := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRatio))), + ) + otel.SetTracerProvider(provider) + return provider.Shutdown, nil +} + +// Tracer is the tracer Warp's components start spans with. +func Tracer() trace.Tracer { return otel.Tracer(instrumentation) } + +// Start begins a span in the current trace. It is a no-op span when +// tracing is off. +func Start(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, trace.Span) { + return Tracer().Start(ctx, name, trace.WithAttributes(attrs...)) +} + +// End finishes a span, recording err on it when there is one. +func End(span trace.Span, err error) { + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + span.End() +} + +// Instrument wraps one route's handler in a server span named after the +// route's pattern — "PUT /_matrix/client/v3/rooms/{roomID}/send/..." — +// rather than its path, which would make every room its own span name. +// +// It has the shape of httputil.Instrumenter, so it composes with the +// metrics middleware at route registration. +func Instrument(route string, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := otel.GetTextMapPropagator().Extract(r.Context(), propagation.HeaderCarrier(r.Header)) + ctx, span := Tracer().Start(ctx, route, + trace.WithSpanKind(trace.SpanKindServer), + trace.WithAttributes( + semconv.HTTPRequestMethodKey.String(r.Method), + semconv.HTTPRoute(route), + semconv.URLPath(r.URL.Path), + semconv.UserAgentOriginal(r.UserAgent()), + )) + defer span.End() + + rec := &httputil.ResponseRecorder{ResponseWriter: w} + next.ServeHTTP(rec, r.WithContext(ctx)) + + status := rec.Status + if status == 0 { + status = http.StatusOK + } + span.SetAttributes(semconv.HTTPResponseStatusCode(status)) + if status >= http.StatusInternalServerError { + span.SetStatus(codes.Error, strconv.Itoa(status)) + } + }) +} + +// Transport wraps an outbound HTTP transport so that every request it sends +// is a client span, carrying the trace context to the other side. A nil +// base is http.DefaultTransport. +func Transport(base http.RoundTripper) http.RoundTripper { + if base == nil { + base = http.DefaultTransport + } + return &transport{base: base} +} + +type transport struct{ base http.RoundTripper } + +func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { + ctx, span := Tracer().Start(req.Context(), req.Method+" "+req.URL.Host, + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes( + semconv.HTTPRequestMethodKey.String(req.Method), + semconv.ServerAddress(req.URL.Hostname()), + semconv.URLPath(req.URL.Path), + )) + defer span.End() + + // RoundTrip must not modify the request it was given. + out := req.Clone(ctx) + otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(out.Header)) + resp, err := t.base.RoundTrip(out) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return nil, err + } + span.SetAttributes(semconv.HTTPResponseStatusCode(resp.StatusCode)) + if resp.StatusCode >= http.StatusInternalServerError { + span.SetStatus(codes.Error, strconv.Itoa(resp.StatusCode)) + } + return resp, nil +} diff --git a/server/internal/tracing/tracing_test.go b/server/internal/tracing/tracing_test.go new file mode 100644 index 0000000..7aacc54 --- /dev/null +++ b/server/internal/tracing/tracing_test.go @@ -0,0 +1,122 @@ +package tracing_test + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "go.opentelemetry.io/otel" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" + + "github.com/dariuszjaniak/warp/server/internal/tracing" +) + +// recordSpans installs a tracer provider that keeps every span in memory. +func recordSpans(t *testing.T) *tracetest.InMemoryExporter { + t.Helper() + if _, err := tracing.Setup(t.Context(), tracing.Config{}); err != nil { + t.Fatalf("Setup: %v", err) + } + exporter := tracetest.NewInMemoryExporter() + provider := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter)) + previous := otel.GetTracerProvider() + otel.SetTracerProvider(provider) + t.Cleanup(func() { + _ = provider.Shutdown(context.Background()) + otel.SetTracerProvider(previous) + }) + return exporter +} + +func TestInstrumentNamesTheSpanAfterTheRoute(t *testing.T) { + exporter := recordSpans(t) + const route = "GET /_matrix/client/v3/rooms/{roomID}/state" + + var inner trace.SpanContext + h := tracing.Instrument(route, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + inner = trace.SpanContextFromContext(r.Context()) + w.WriteHeader(http.StatusTeapot) + })) + + // A request that arrives inside a trace continues it. + req := httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/rooms/!a:b/state", nil) + req.Header.Set("traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01") + h.ServeHTTP(httptest.NewRecorder(), req) + + spans := exporter.GetSpans() + if len(spans) != 1 { + t.Fatalf("recorded %d spans", len(spans)) + } + span := spans[0] + if span.Name != route { + t.Errorf("span name = %q", span.Name) + } + if span.SpanKind != trace.SpanKindServer { + t.Errorf("span kind = %v", span.SpanKind) + } + if got := span.SpanContext.TraceID().String(); got != "4bf92f3577b34da6a3ce929d0e0e4736" { + t.Errorf("trace ID = %s, want the caller's", got) + } + if !inner.IsValid() || inner.SpanID() != span.SpanContext.SpanID() { + t.Error("the handler did not run inside the span") + } + var status int64 + for _, a := range span.Attributes { + if a.Key == "http.response.status_code" { + status = a.Value.AsInt64() + } + } + if status != http.StatusTeapot { + t.Errorf("status attribute = %d", status) + } +} + +func TestTransportCarriesTheTraceOn(t *testing.T) { + exporter := recordSpans(t) + + var traceparent string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + traceparent = r.Header.Get("traceparent") + })) + defer upstream.Close() + + ctx, parent := tracing.Start(t.Context(), "parent") + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, upstream.URL+"/x", nil) + resp, err := (&http.Client{Transport: tracing.Transport(nil)}).Do(req) + if err != nil { + t.Fatalf("request: %v", err) + } + _ = resp.Body.Close() + tracing.End(parent, nil) + + if traceparent == "" { + t.Fatal("the outbound request carried no traceparent") + } + if req.Header.Get("traceparent") != "" { + t.Error("the caller's request was modified") + } + if spans := exporter.GetSpans(); len(spans) != 2 { + t.Errorf("recorded %d spans, want the client span and its parent", len(spans)) + } +} + +func TestDisabledTracingIsANoOp(t *testing.T) { + stop, err := tracing.Setup(t.Context(), tracing.Config{Enabled: false}) + if err != nil { + t.Fatalf("Setup: %v", err) + } + _, span := tracing.Start(t.Context(), "nothing") + if span.SpanContext().IsSampled() { + t.Error("a span was sampled with tracing off") + } + tracing.End(span, nil) + if err := stop(t.Context()); err != nil { + t.Errorf("stop: %v", err) + } + if _, err := tracing.Setup(t.Context(), tracing.Config{Enabled: true, Endpoint: "not a url"}); err == nil { + t.Error("a malformed endpoint was accepted") + } +} diff --git a/warp.yaml.example b/warp.yaml.example index a801c9d..623c778 100644 --- a/warp.yaml.example +++ b/warp.yaml.example @@ -51,11 +51,13 @@ listen: database: # sqlite (dev, single user), postgres (production) or pocketbase - # (appliance; not implemented yet). WARP_DB_DRIVER + # (appliance: sqlite inside an embedded PocketBase, media on its + # filesystem, local or S3). WARP_DB_DRIVER driver: sqlite # sqlite: file:warp.db?_pragma=journal_mode(WAL) # postgres: postgres://warp:secret@localhost/warp?sslmode=disable + # pocketbase: the data directory, e.g. ./pb_data # WARP_DB_CONNECTION_STRING connection_string: "file:warp.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_pragma=foreign_keys(1)" @@ -67,6 +69,19 @@ database: # behind an explicit `warp migrate`. WARP_DB_AUTO_MIGRATE auto_migrate: true + # The pocketbase driver only. With S3 on, media content goes to the + # bucket instead of /storage; media.base_path is not + # used by this driver either way. + pocketbase: + s3: + enabled: false # WARP_PB_S3_ENABLED + bucket: "" # WARP_PB_S3_BUCKET + region: "" # WARP_PB_S3_REGION + endpoint: "" # WARP_PB_S3_ENDPOINT, e.g. https://s3.eu-central-1.amazonaws.com + access_key: "" # WARP_PB_S3_ACCESS_KEY + secret: "" # WARP_PB_S3_SECRET + force_path_style: false # WARP_PB_S3_FORCE_PATH_STYLE (MinIO) + client_api: # The Matrix spec expects a homeserver to allow any origin. Narrow this # if only your own web client talks to it. WARP_CORS_ALLOWED_ORIGINS @@ -294,3 +309,12 @@ logging: metrics: enabled: true # WARP_METRICS_ENABLED path: /metrics # WARP_METRICS_PATH + +# OpenTelemetry tracing, exported over OTLP/HTTP to a collector, Jaeger, +# Tempo or anything else that speaks it. See docs/deployment.md. +tracing: + enabled: false # WARP_TRACING_ENABLED + # Empty falls back to OTEL_EXPORTER_OTLP_ENDPOINT, then localhost:4318. + endpoint: "" # WARP_TRACING_ENDPOINT, e.g. http://localhost:4318 + sample_ratio: 1 # WARP_TRACING_SAMPLE_RATIO, 0 to 1 + service_name: warp # WARP_TRACING_SERVICE_NAME