Phase 10: hardening and ops
CI / Test (ubuntu-latest) (push) Canceled after 0s
CI / Test (windows-latest) (push) Canceled after 0s
CI / Storage tests on PostgreSQL (push) Canceled after 0s
CI / Complement (federation off) (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
CI / Smoke test the binary (push) Canceled after 0s

Implements PLAN.md Phase 10.

Simplified Sliding Sync (MSC4186) is served at the unstable
org.matrix.simplified_msc3575 endpoint and advertised in /versions. Lists
are sorted by activity and filtered by DM, invite, encryption, room type,
space and tag; room subscriptions, required_state with wildcards, $ME and
$LAZY, and the to_device, e2ee, account_data, receipts and typing
extensions are supported. Per-connection state is kept in memory and an
unknown pos answers M_UNKNOWN_POS.

Threads and relations: m.relates_to is indexed on roomserver_events
(migration 0012), /relations (with recursion) and /threads are served, and
every event a timeline read returns carries its m.thread, m.replace and
m.reference summaries in unsigned.m.relations. Notifications carry a
thread ID (migration 0013): threaded receipts clear only their thread, and
sync reports unread_thread_notifications when the filter asks.

Spaces: /hierarchy walks a space breadth first, asking other servers about
rooms this one is not in through the new federation hierarchy endpoint;
room_summary describes one room. /search runs on FTS5 (sqlite, pocketbase)
or tsvector (postgres), bounded to what each user may see, with context,
profiles, state, grouping and paging.

The PocketBase driver is real: the sqlite schema and store on an embedded
PocketBase app's database, with media in its filesystem, local or S3
(database.pocketbase.s3). It runs the full storage test suite.

Ops: OpenTelemetry tracing over OTLP/HTTP (tracing.*), with server spans
per route, component spans, and trace propagation on federation, push and
application service calls. New metrics for persisted events, waiting
syncs, sliding sync connections and the database pool, with a Grafana
dashboard. deploy/ adds a Compose setup (PostgreSQL, Prometheus, Grafana,
Jaeger) and a Helm chart; cmd/warp-loadtest loads a server and reports
request and delivery latencies.

The load tester found a long-standing bug, fixed here: on a server where
no account data had been written, every incremental /sync re-sent the push
rules and returned at once, and an all-zero token was taken for an initial
sync, so long polls became busy loops.

The PostgreSQL migrations and search dialect compile but were not run
against a live PostgreSQL in this change.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
tajniak81andClaude Opus 5.5 committed 2026-10-17 20:21:31 +02:00
1 parent a94794b866
commit 487e0f410b
100 files changed
+10528 -893

No files matched your search

+514
View File
@@ -0,0 +1,514 @@
// Command warp-loadtest drives a homeserver the way a crowd of clients
// would, and reports how it held up.
//
// It registers users, creates rooms and has every user join them, then for
// the length of the run each user sends messages at a steady rate while
// syncing in a long-poll loop, the way a client sits in /sync. Every message
// carries the time it was sent, so a user's sync can measure how long the
// server took to deliver what somebody else said — the number a user of a
// chat server actually feels — alongside the latency of each request.
//
// It speaks the plain Client-Server API and works against any homeserver,
// not only Warp. Point it at a test server: it creates accounts and rooms
// it never cleans up. Rate limiting on the server has to be off, or the
// run measures the rate limiter.
//
// warp-loadtest -url http://localhost:8008 -users 50 -rooms 10 -rate 0.5 -duration 2m
package main
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net/http"
"net/url"
"os"
"os/signal"
"slices"
"strconv"
"strings"
"sync"
"sync/atomic"
"text/tabwriter"
"time"
)
type options struct {
baseURL string
users int
rooms int
rate float64
duration time.Duration
prefix string
password string
regToken string
sliding bool
setupParallel int
}
func main() {
var o options
flag.StringVar(&o.baseURL, "url", "http://localhost:8008", "the homeserver's client API base URL")
flag.IntVar(&o.users, "users", 20, "how many users to register")
flag.IntVar(&o.rooms, "rooms", 5, "how many rooms to create; every user joins every room")
flag.Float64Var(&o.rate, "rate", 0.2, "messages per second per user")
flag.DurationVar(&o.duration, "duration", time.Minute, "how long to send for")
flag.StringVar(&o.prefix, "prefix", "", "localpart prefix for the users (default: random)")
flag.StringVar(&o.password, "password", "loadtest-password-1234", "the users' password")
flag.StringVar(&o.regToken, "registration-token", "", "a registration token, for a server that requires one")
flag.BoolVar(&o.sliding, "sliding", false, "sync with Simplified Sliding Sync instead of /sync")
flag.IntVar(&o.setupParallel, "setup-parallelism", 8, "concurrent requests while registering and joining")
flag.Parse()
if o.prefix == "" {
var b [3]byte
_, _ = rand.Read(b[:])
o.prefix = "lt" + hex.EncodeToString(b[:])
}
o.baseURL = strings.TrimRight(o.baseURL, "/")
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
defer stop()
if err := run(ctx, o); err != nil {
fmt.Fprintln(os.Stderr, "warp-loadtest:", err)
os.Exit(1)
}
}
func run(ctx context.Context, o options) error {
stats := newStats()
c := &client{base: o.baseURL, http: &http.Client{Timeout: 90 * time.Second}, stats: stats}
fmt.Printf("registering %d users as %s-N on %s\n", o.users, o.prefix, o.baseURL)
users := make([]*user, o.users)
if err := parallel(ctx, o.users, o.setupParallel, func(i int) error {
u, err := c.register(ctx, fmt.Sprintf("%s-%d", o.prefix, i), o.password, o.regToken)
users[i] = u
return err
}); err != nil {
return err
}
fmt.Printf("creating %d rooms and joining every user to each\n", o.rooms)
rooms := make([]string, o.rooms)
for i := range rooms {
roomID, err := c.createRoom(ctx, users[0], fmt.Sprintf("load test %s #%d", o.prefix, i))
if err != nil {
return err
}
rooms[i] = roomID
}
if err := parallel(ctx, len(users)-1, o.setupParallel, func(i int) error {
for _, roomID := range rooms {
if err := c.join(ctx, users[i+1], roomID); err != nil {
return err
}
}
return nil
}); err != nil {
return err
}
fmt.Printf("sending %.2f msg/s per user for %s (%s)\n", o.rate, o.duration, map[bool]string{true: "sliding sync", false: "/sync"}[o.sliding])
var wg sync.WaitGroup
syncCtx, stopSync := context.WithCancel(ctx)
for _, u := range users {
wg.Add(1)
go func() {
defer wg.Done()
if o.sliding {
c.slidingLoop(syncCtx, u, len(rooms))
} else {
c.syncLoop(syncCtx, u)
}
}()
}
// The syncs need a moment to reach the top of the stream before the
// first message, or the first delivery latencies would include the
// initial sync.
time.Sleep(2 * time.Second)
runCtx, cancel := context.WithTimeout(ctx, o.duration)
defer cancel()
var senders sync.WaitGroup
start := time.Now()
for i, u := range users {
senders.Add(1)
go func() {
defer senders.Done()
c.sendLoop(runCtx, u, rooms, o.rate, float64(i)/float64(len(users)))
}()
}
senders.Wait()
elapsed := time.Since(start)
// Give the last messages time to be delivered before the syncs stop.
time.Sleep(3 * time.Second)
stopSync()
wg.Wait()
stats.report(os.Stdout, elapsed)
return nil
}
// parallel runs fn for 0..n-1, at most width at once, stopping at the
// first error.
func parallel(ctx context.Context, n, width int, fn func(int) error) error {
sem := make(chan struct{}, max(width, 1))
var (
wg sync.WaitGroup
mu sync.Mutex
firstErr error
)
for i := range n {
if ctx.Err() != nil {
break
}
sem <- struct{}{}
wg.Add(1)
go func() {
defer func() { <-sem; wg.Done() }()
if err := fn(i); err != nil {
mu.Lock()
if firstErr == nil {
firstErr = err
}
mu.Unlock()
}
}()
}
wg.Wait()
if firstErr != nil {
return firstErr
}
return ctx.Err()
}
type user struct {
id string
token string
}
type client struct {
base string
http *http.Client
stats *stats
txn atomic.Int64
}
// do sends one request, timing it under op, and decodes the response into
// out when there is one.
func (c *client) do(ctx context.Context, op, method, path, token string, body, out any) (int, error) {
var reader io.Reader
if body != nil {
raw, err := json.Marshal(body)
if err != nil {
return 0, err
}
reader = bytes.NewReader(raw)
}
req, err := http.NewRequestWithContext(ctx, method, c.base+path, reader)
if err != nil {
return 0, err
}
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
if token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}
start := time.Now()
resp, err := c.http.Do(req)
if err != nil {
if ctx.Err() == nil {
c.stats.fail(op)
}
return 0, err
}
defer func() { _ = resp.Body.Close() }()
raw, err := io.ReadAll(resp.Body)
c.stats.observe(op, time.Since(start), resp.StatusCode < 400 || resp.StatusCode == http.StatusUnauthorized && op == "register")
if err != nil {
return resp.StatusCode, err
}
if out != nil && len(raw) > 0 {
_ = json.Unmarshal(raw, out)
}
if resp.StatusCode >= 400 && (resp.StatusCode != http.StatusUnauthorized || op != "register") {
return resp.StatusCode, fmt.Errorf("%s %s: %d %s", method, path, resp.StatusCode, strings.TrimSpace(string(raw)))
}
return resp.StatusCode, nil
}
// register walks the user-interactive registration flow, answering the
// dummy and registration-token stages.
func (c *client) register(ctx context.Context, localpart, password, token string) (*user, error) {
body := map[string]any{"username": localpart, "password": password, "inhibit_login": false}
var challenge struct {
Session string `json:"session"`
Flows []struct {
Stages []string `json:"stages"`
} `json:"flows"`
UserID string `json:"user_id"`
AccessToken string `json:"access_token"`
}
if _, err := c.do(ctx, "register", http.MethodPost, "/_matrix/client/v3/register", "", body, &challenge); err != nil {
return nil, err
}
if challenge.AccessToken != "" {
return &user{id: challenge.UserID, token: challenge.AccessToken}, nil
}
if len(challenge.Flows) == 0 {
return nil, errors.New("register: the server offered no flow")
}
for _, stage := range challenge.Flows[0].Stages {
auth := map[string]any{"type": stage, "session": challenge.Session}
switch stage {
case "m.login.dummy":
case "m.login.registration_token":
if token == "" {
return nil, errors.New("register: the server wants a registration token; pass -registration-token")
}
auth["token"] = token
default:
return nil, fmt.Errorf("register: cannot complete the %s stage", stage)
}
body["auth"] = auth
var next struct {
UserID string `json:"user_id"`
AccessToken string `json:"access_token"`
}
if _, err := c.do(ctx, "register", http.MethodPost, "/_matrix/client/v3/register", "", body, &next); err != nil {
return nil, err
}
if next.AccessToken != "" {
return &user{id: next.UserID, token: next.AccessToken}, nil
}
}
return nil, errors.New("register: the flow ended without an access token")
}
func (c *client) createRoom(ctx context.Context, u *user, name string) (string, error) {
var out struct {
RoomID string `json:"room_id"`
}
_, err := c.do(ctx, "createRoom", http.MethodPost, "/_matrix/client/v3/createRoom", u.token,
map[string]any{"name": name, "preset": "public_chat"}, &out)
return out.RoomID, err
}
func (c *client) join(ctx context.Context, u *user, roomID string) error {
_, err := c.do(ctx, "join", http.MethodPost, "/_matrix/client/v3/join/"+url.PathEscape(roomID), u.token, map[string]any{}, nil)
return err
}
// sentMarker prefixes a message body with the time it was sent.
const sentMarker = "warp-loadtest sent="
// sendLoop sends at a steady rate until the context ends. Users start
// staggered across the first interval — phase is this user's place in it,
// from 0 to 1 — so that the load is a flow rather than a pulse.
func (c *client) sendLoop(ctx context.Context, u *user, rooms []string, rate, phase float64) {
if rate <= 0 {
return
}
interval := time.Duration(float64(time.Second) / rate)
select {
case <-ctx.Done():
return
case <-time.After(time.Duration(phase * float64(interval))):
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
n := 0
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
}
roomID := rooms[n%len(rooms)]
n++
body := sentMarker + strconv.FormatInt(time.Now().UnixNano(), 10)
path := fmt.Sprintf("/_matrix/client/v3/rooms/%s/send/m.room.message/lt%d",
url.PathEscape(roomID), c.txn.Add(1))
_, _ = c.do(ctx, "send", http.MethodPut, path, u.token, map[string]any{"msgtype": "m.text", "body": body}, nil)
}
}
// syncLoop sits in /sync, measuring how long each message took to arrive.
func (c *client) syncLoop(ctx context.Context, u *user) {
since := ""
for ctx.Err() == nil {
path := "/_matrix/client/v3/sync?timeout=30000"
if since != "" {
path += "&since=" + url.QueryEscape(since)
}
var resp struct {
NextBatch string `json:"next_batch"`
Rooms struct {
Join map[string]struct {
Timeline struct {
Events []timelineEvent `json:"events"`
} `json:"timeline"`
} `json:"join"`
} `json:"rooms"`
}
op := "sync"
if since == "" {
op = "initial sync"
}
if _, err := c.do(ctx, op, http.MethodGet, path, u.token, nil, &resp); err != nil {
if ctx.Err() == nil {
time.Sleep(time.Second)
}
continue
}
received := time.Now()
if since != "" {
for _, room := range resp.Rooms.Join {
c.deliveries(u, room.Timeline.Events, received)
}
}
since = resp.NextBatch
}
}
// slidingLoop is syncLoop on Simplified Sliding Sync.
func (c *client) slidingLoop(ctx context.Context, u *user, rooms int) {
const path = "/_matrix/client/unstable/org.matrix.simplified_msc3575/sync"
body := map[string]any{
"lists": map[string]any{"all": map[string]any{
"ranges": [][2]int{{0, max(rooms-1, 0)}}, "timeline_limit": 20,
"required_state": [][]string{{"m.room.name", ""}},
}},
}
pos := ""
for ctx.Err() == nil {
q := "?timeout=30000"
if pos != "" {
q += "&pos=" + url.QueryEscape(pos)
}
var resp struct {
Pos string `json:"pos"`
Rooms map[string]struct {
Timeline []timelineEvent `json:"timeline"`
} `json:"rooms"`
}
op := "sliding sync"
if pos == "" {
op = "initial sliding sync"
}
status, err := c.do(ctx, op, http.MethodPost, path+q, u.token, body, &resp)
if err != nil {
if status == http.StatusBadRequest {
pos = "" // M_UNKNOWN_POS: start again
}
if ctx.Err() == nil {
time.Sleep(time.Second)
}
continue
}
received := time.Now()
if pos != "" {
for _, room := range resp.Rooms {
c.deliveries(u, room.Timeline, received)
}
}
pos = resp.Pos
}
}
type timelineEvent struct {
Type string `json:"type"`
Sender string `json:"sender"`
Content struct {
Body string `json:"body"`
} `json:"content"`
}
// deliveries records how long each message from somebody else took to
// reach this user.
func (c *client) deliveries(u *user, events []timelineEvent, received time.Time) {
for _, e := range events {
if e.Type != "m.room.message" || e.Sender == u.id {
continue
}
raw, ok := strings.CutPrefix(e.Content.Body, sentMarker)
if !ok {
continue
}
sent, err := strconv.ParseInt(raw, 10, 64)
if err != nil {
continue
}
c.stats.observe("delivery", received.Sub(time.Unix(0, sent)), true)
}
}
// stats collects latencies per operation.
type stats struct {
mu sync.Mutex
ops map[string][]time.Duration
errors map[string]int
}
func newStats() *stats {
return &stats{ops: map[string][]time.Duration{}, errors: map[string]int{}}
}
func (s *stats) observe(op string, d time.Duration, ok bool) {
s.mu.Lock()
defer s.mu.Unlock()
s.ops[op] = append(s.ops[op], d)
if !ok {
s.errors[op]++
}
}
func (s *stats) fail(op string) {
s.mu.Lock()
defer s.mu.Unlock()
s.errors[op]++
}
func (s *stats) report(w io.Writer, elapsed time.Duration) {
s.mu.Lock()
defer s.mu.Unlock()
names := make([]string, 0, len(s.ops))
for name := range s.ops {
names = append(names, name)
}
slices.Sort(names)
fmt.Fprintf(w, "\nran for %s\n\n", elapsed.Round(time.Millisecond))
tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', tabwriter.AlignRight)
fmt.Fprintln(tw, "operation\tcount\terrors\trate/s\tp50\tp90\tp99\tmax\t")
for _, name := range names {
d := s.ops[name]
slices.Sort(d)
rate := float64(len(d)) / elapsed.Seconds()
fmt.Fprintf(tw, "%s\t%d\t%d\t%.1f\t%s\t%s\t%s\t%s\t\n", name, len(d), s.errors[name], rate,
pct(d, 0.5), pct(d, 0.9), pct(d, 0.99), pct(d, 1))
}
_ = tw.Flush()
fmt.Fprintln(w, "\nsync latencies include the long-poll wait; delivery is send-to-receipt across users.")
}
func pct(sorted []time.Duration, q float64) string {
if len(sorted) == 0 {
return "-"
}
i := int(q*float64(len(sorted))+0.5) - 1
i = max(0, min(i, len(sorted)-1))
return sorted[i].Round(100 * time.Microsecond).String()
}
+1 -7
View File
@@ -102,13 +102,7 @@ func runCreateAccount(ctx context.Context, args []string) error {
// openDatabase opens the storage driver named in the config.
func openDatabase(ctx context.Context, cfg *config.Config, log *slog.Logger) (storage.Database, error) {
return storage.Open(ctx, storage.Config{
Driver: cfg.Database.Driver,
ConnectionString: cfg.Database.ConnectionString,
MaxOpenConns: cfg.Database.MaxOpenConns,
MaxIdleConns: cfg.Database.MaxIdleConns,
ConnMaxLifetime: cfg.Database.ConnMaxLifetime,
}, log)
return storage.Open(ctx, cfg.Database.Storage(), log)
}
// resolvePassword settles where the password comes from.
+31 -1
View File
@@ -4,7 +4,13 @@ go 1.26.4
require (
github.com/jackc/pgx/v5 v5.11.0
github.com/pocketbase/dbx v1.12.0
github.com/pocketbase/pocketbase v0.39.11
github.com/prometheus/client_golang v1.25.0
go.opentelemetry.io/otel v1.47.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0
go.opentelemetry.io/otel/sdk v1.47.0
go.opentelemetry.io/otel/trace v1.47.0
golang.org/x/crypto v0.58.0
golang.org/x/image v0.46.0
golang.org/x/sync v0.24.0
@@ -14,24 +20,48 @@ require (
)
require (
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/disintegration/imaging v1.6.2 // indirect
github.com/domodwyer/mailyak/v3 v3.6.2 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fatih/color v1.19.0 // indirect
github.com/fsnotify/fsnotify v1.10.1 // indirect
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
github.com/ganigeorgiev/fexpr v0.6.0 // indirect
github.com/go-logr/logr v1.4.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/mattn/go-colorable v0.1.15 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/pocketbase/ozzo-validation/v4 v4.3.0 // indirect
github.com/prometheus/client_model v0.6.3 // indirect
github.com/prometheus/common v0.72.0 // indirect
github.com/prometheus/procfs v0.22.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rogpeppe/go-internal v1.16.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect
go.opentelemetry.io/otel/log v1.47.0 // indirect
go.opentelemetry.io/otel/metric v1.47.0 // indirect
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
golang.org/x/net v0.60.0 // indirect
golang.org/x/oauth2 v0.37.0 // indirect
golang.org/x/sys v0.49.0 // indirect
golang.org/x/text v0.43.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect
google.golang.org/grpc v1.83.2 // indirect
google.golang.org/protobuf v1.36.12 // indirect
modernc.org/libc v1.77.1 // indirect
modernc.org/mathutil v1.7.1 // indirect
+88 -3
View File
@@ -1,17 +1,49 @@
github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8=
github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/ganigeorgiev/fexpr v0.6.0 h1:Fza3O/QMBKEudUvxV862qe6GjxM60GJjjKytdp+VQus=
github.com/ganigeorgiev/fexpr v0.6.0/go.mod h1:RyGiGqmeXhEQ6+mlGdnUleLHgtzzu/VGO2WtJkF5drE=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
@@ -24,12 +56,14 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
@@ -37,6 +71,12 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pocketbase/dbx v1.12.0 h1:/oLErM+A0b4xI0PWTGPqSDVjzix48PqI/bng2l0PzoA=
github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs=
github.com/pocketbase/ozzo-validation/v4 v4.3.0 h1:uKBDVma7bZqgR2a6AwE+k9hkuDFfiZMpBHQdZ1z3iQs=
github.com/pocketbase/ozzo-validation/v4 v4.3.0/go.mod h1:6XNjSTw/Jb2F8LOkKO3oyzIWExbrGiYoS4uVxVwz90g=
github.com/pocketbase/pocketbase v0.39.11 h1:cl/Kh13ukof/4BAEku3OozYrLl85M5/bmH62Ny4szFc=
github.com/pocketbase/pocketbase v0.39.11/go.mod h1:5CaCvp/52fZJ5/qyYsqpCGyVue/kjez889cQAATD2cY=
github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKqbYtNOXuiK0=
github.com/prometheus/client_golang v1.25.0/go.mod h1:j2X/4/kaQHLPnmyEU8zjmkndo1T317VpCIQEml8LTKk=
github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
@@ -49,38 +89,83 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g=
github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM=
go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38=
go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI=
go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw=
go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY=
go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU=
go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I=
go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0=
go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw=
go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw=
go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM=
go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0=
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ=
golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew=
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI=
golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE=
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc=
google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+3
View File
@@ -13,6 +13,8 @@ import (
"strconv"
"strings"
"time"
"github.com/dariuszjaniak/warp/server/internal/tracing"
)
// Errors of the Application Service API client. The ping endpoint reports
@@ -57,6 +59,7 @@ type Client struct {
func NewClient(timeout time.Duration) *Client {
return &Client{
http: &http.Client{
Transport: tracing.Transport(nil),
// A service is reached at the URL its registration names and
// nowhere else: a redirect would take the hs_token with it.
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+14
View File
@@ -313,9 +313,19 @@ func (c *ClientAPI) registerSyncRoutes(r *httputil.Router) {
}
c.authed(r, "GET "+v3+"/sync", c.limitSync, c.handleSync)
c.authed(r, "POST "+slidingSyncPath, c.limitSync, c.handleSlidingSync)
c.authed(r, "GET "+v3+"/rooms/{roomID}/messages", c.limitDefault, c.handleMessages)
c.authed(r, "GET "+v3+"/rooms/{roomID}/context/{eventID}", c.limitDefault, c.handleContext)
// Relations and threads, which the spec introduced at v1.
c.authed(r, "GET "+v1+"/rooms/{roomID}/relations/{eventID}", c.limitDefault, c.handleRelations)
c.authed(r, "GET "+v1+"/rooms/{roomID}/relations/{eventID}/{relType}", c.limitDefault, c.handleRelations)
c.authed(r, "GET "+v1+"/rooms/{roomID}/relations/{eventID}/{relType}/{eventType}",
c.limitDefault, c.handleRelations)
c.authed(r, "GET "+v1+"/rooms/{roomID}/threads", c.limitDefault, c.handleThreads)
c.authed(r, "POST "+v3+"/search", c.limitDefault, c.handleSearch)
// Filters.
c.authed(r, "POST "+v3+"/user/{userID}/filter", c.limitDefault, c.handlePostFilter)
c.authed(r, "GET "+v3+"/user/{userID}/filter/{filterID}", c.limitDefault, c.handleGetFilter)
@@ -387,6 +397,10 @@ func (c *ClientAPI) registerRoomRoutes(r *httputil.Router) {
c.limitDefault, c.handleGetJoinedMembers)
c.authed(r, "GET "+v3+"/joined_rooms", c.limitDefault, c.handleGetJoinedRooms)
// Spaces, and the summary of a room the user may not be in.
c.authed(r, "GET "+v1+"/rooms/{roomID}/hierarchy", c.limitDefault, c.handleHierarchy)
c.authed(r, "GET "+v1+"/room_summary/{roomIDOrAlias}", c.limitDefault, c.handleRoomSummary)
// The directory. Resolving an alias and reading the public room list
// need no token: a client handed a link has to be able to follow it
// before it decides whether to log in.
+88
View File
@@ -0,0 +1,88 @@
package clientapi
import (
"net/http"
"github.com/dariuszjaniak/warp/server/internal/httputil"
"github.com/dariuszjaniak/warp/server/internal/storage"
"github.com/dariuszjaniak/warp/server/internal/syncapi"
"github.com/dariuszjaniak/warp/server/internal/userapi"
)
// handleRelations implements
// GET /_matrix/client/v1/rooms/{roomID}/relations/{eventID}[/{relType}[/{eventType}]].
//
// The three routes share the handler; the narrower ones only fill in more
// of the path.
func (c *ClientAPI) handleRelations(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
roomID, err := pathRoomID(r)
if err != nil {
return err
}
eventID := r.PathValue("eventID")
if eventID == "" {
return httputil.MissingParam("An event ID is required")
}
query := r.URL.Query()
direction := storage.DirectionBackward
switch query.Get("dir") {
case "f":
direction = storage.DirectionForward
case "b", "":
default:
return httputil.InvalidParam("dir must be \"b\" or \"f\"")
}
limit, err := intParam(query.Get("limit"))
if err != nil {
return err
}
recurse, err := boolParam(query.Get("recurse"))
if err != nil {
return err
}
resp, err := c.sync.Relations(r.Context(), syncapi.RelationsRequest{
RoomID: roomID,
EventID: eventID,
User: auth.UserID,
RelType: r.PathValue("relType"),
EventType: r.PathValue("eventType"),
From: query.Get("from"),
To: query.Get("to"),
Direction: direction,
Limit: limit,
Recurse: recurse,
})
if err != nil {
return syncError(err)
}
httputil.WriteJSON(w, http.StatusOK, resp)
return nil
}
// handleThreads implements GET /_matrix/client/v1/rooms/{roomID}/threads.
func (c *ClientAPI) handleThreads(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
roomID, err := pathRoomID(r)
if err != nil {
return err
}
query := r.URL.Query()
limit, err := intParam(query.Get("limit"))
if err != nil {
return err
}
resp, err := c.sync.Threads(r.Context(), syncapi.ThreadsRequest{
RoomID: roomID,
User: auth.UserID,
Include: query.Get("include"),
From: query.Get("from"),
Limit: limit,
})
if err != nil {
return syncError(err)
}
httputil.WriteJSON(w, http.StatusOK, resp)
return nil
}
+306
View File
@@ -0,0 +1,306 @@
package clientapi_test
import (
"fmt"
"net/http"
"net/url"
"testing"
"time"
)
// Relations and threads: what /relations lists, what /threads lists, and
// the summaries a parent carries in unsigned.m.relations wherever it is
// served.
// sendEvent sends an event of any type and returns its ID.
func (ts *testServer) sendEvent(acc account, roomID, eventType string, content map[string]any) string {
ts.t.Helper()
path := fmt.Sprintf("/_matrix/client/v3/rooms/%s/send/%s/%s",
url.PathEscape(roomID), url.PathEscape(eventType), fmt.Sprintf("txn-%d", time.Now().UnixNano()))
rec := ts.put(path, acc.AccessToken, content)
expectStatus(ts.t, rec, http.StatusOK)
eventID, _ := jsonBody(ts.t, rec)["event_id"].(string)
if eventID == "" {
ts.t.Fatalf("send returned no event ID: %s", rec.Body.String())
}
return eventID
}
// threadReply sends a message in a thread.
func (ts *testServer) threadReply(acc account, roomID, rootID, body string) string {
ts.t.Helper()
return ts.sendEvent(acc, roomID, "m.room.message", map[string]any{
"msgtype": "m.text", "body": body,
"m.relates_to": map[string]any{"rel_type": "m.thread", "event_id": rootID},
})
}
type relationsResult struct {
Chunk []syncEvent `json:"chunk"`
NextBatch string `json:"next_batch"`
RecursionDepth int `json:"recursion_depth"`
}
func (ts *testServer) relations(acc account, roomID, path string, query url.Values) relationsResult {
ts.t.Helper()
full := "/_matrix/client/v1/rooms/" + url.PathEscape(roomID) + "/relations/" + path
if encoded := query.Encode(); encoded != "" {
full += "?" + encoded
}
rec := ts.get(full, acc.AccessToken)
expectStatus(ts.t, rec, http.StatusOK)
return decode[relationsResult](ts.t, rec)
}
// threadSummaryOf digs unsigned.m.relations.m.thread out of an event.
func threadSummaryOf(t *testing.T, event map[string]any) map[string]any {
t.Helper()
unsigned, _ := event["unsigned"].(map[string]any)
relations, _ := unsigned["m.relations"].(map[string]any)
summary, _ := relations["m.thread"].(map[string]any)
return summary
}
func TestRelationsListsChildrenNewestFirst(t *testing.T) {
ts := newTestServer(t, nil)
alice := ts.register("alice", "correct-horse-battery-staple")
roomID := ts.createRoom(alice, nil)
root := ts.sendMessage(alice, roomID, "root")
r1 := ts.threadReply(alice, roomID, root, "one")
r2 := ts.threadReply(alice, roomID, root, "two")
reaction := ts.sendEvent(alice, roomID, "m.reaction", map[string]any{
"m.relates_to": map[string]any{"rel_type": "m.annotation", "event_id": root, "key": "👍"},
})
ts.sendMessage(alice, roomID, "unrelated")
all := ts.relations(alice, roomID, url.PathEscape(root), nil)
if got := idsOf(all.Chunk); !equalStrings(got, []string{reaction, r2, r1}) {
t.Fatalf("relations = %v, want the reaction and the two replies newest first", got)
}
threads := ts.relations(alice, roomID, url.PathEscape(root)+"/m.thread", nil)
if got := idsOf(threads.Chunk); !equalStrings(got, []string{r2, r1}) {
t.Errorf("m.thread relations = %v, want the two replies", got)
}
reactions := ts.relations(alice, roomID, url.PathEscape(root)+"/m.annotation/m.reaction", nil)
if got := idsOf(reactions.Chunk); !equalStrings(got, []string{reaction}) {
t.Errorf("m.annotation m.reaction relations = %v, want the reaction", got)
}
// Paging one at a time walks every child and stops.
var paged []string
from := ""
for range 10 {
query := url.Values{"limit": {"1"}}
if from != "" {
query.Set("from", from)
}
page := ts.relations(alice, roomID, url.PathEscape(root), query)
paged = append(paged, idsOf(page.Chunk)...)
if page.NextBatch == "" {
break
}
from = page.NextBatch
}
if !equalStrings(paged, []string{reaction, r2, r1}) {
t.Errorf("paged relations = %v", paged)
}
}
func TestRelationsRecurse(t *testing.T) {
ts := newTestServer(t, nil)
alice := ts.register("alice", "correct-horse-battery-staple")
roomID := ts.createRoom(alice, nil)
root := ts.sendMessage(alice, roomID, "root")
reply := ts.threadReply(alice, roomID, root, "reply")
edit := ts.sendEvent(alice, roomID, "m.room.message", map[string]any{
"msgtype": "m.text", "body": "* reply!",
"m.new_content": map[string]any{"msgtype": "m.text", "body": "reply!"},
"m.relates_to": map[string]any{"rel_type": "m.replace", "event_id": reply},
})
flat := ts.relations(alice, roomID, url.PathEscape(root), nil)
if got := idsOf(flat.Chunk); !equalStrings(got, []string{reply}) {
t.Errorf("non-recursive relations = %v, want only the reply", got)
}
deep := ts.relations(alice, roomID, url.PathEscape(root), url.Values{"recurse": {"true"}})
if got := idsOf(deep.Chunk); !equalStrings(got, []string{edit, reply}) {
t.Errorf("recursive relations = %v, want the edit of the reply too", got)
}
if deep.RecursionDepth != 2 {
t.Errorf("recursion_depth = %d, want 2", deep.RecursionDepth)
}
}
func TestBundledAggregations(t *testing.T) {
ts := newTestServer(t, nil)
alice := ts.register("alice", "correct-horse-battery-staple")
bob := ts.register("bob", "correct-horse-battery-staple")
roomID := ts.createRoom(alice, map[string]any{"preset": "public_chat"})
escaped := url.PathEscape(roomID)
expectStatus(t, ts.post("/_matrix/client/v3/join/"+escaped, bob.AccessToken, map[string]any{}), http.StatusOK)
root := ts.sendMessage(alice, roomID, "root")
ts.threadReply(bob, roomID, root, "first")
latest := ts.threadReply(bob, roomID, root, "second")
edit := ts.sendEvent(alice, roomID, "m.room.message", map[string]any{
"msgtype": "m.text", "body": "* root, edited",
"m.new_content": map[string]any{"msgtype": "m.text", "body": "root, edited"},
"m.relates_to": map[string]any{"rel_type": "m.replace", "event_id": root},
})
// An edit by somebody else is not an edit.
ts.sendEvent(bob, roomID, "m.room.message", map[string]any{
"msgtype": "m.text", "body": "* hijacked",
"m.new_content": map[string]any{"msgtype": "m.text", "body": "hijacked"},
"m.relates_to": map[string]any{"rel_type": "m.replace", "event_id": root},
})
check := func(who string, acc account, participated bool) {
t.Helper()
rec := ts.get("/_matrix/client/v3/rooms/"+escaped+"/event/"+url.PathEscape(root), acc.AccessToken)
expectStatus(t, rec, http.StatusOK)
event := jsonBody(t, rec)
summary := threadSummaryOf(t, event)
if summary == nil {
t.Fatalf("%s: the root has no m.thread summary: %v", who, event["unsigned"])
}
if summary["count"] != float64(2) {
t.Errorf("%s: thread count = %v, want 2", who, summary["count"])
}
latestEvent, _ := summary["latest_event"].(map[string]any)
if latestEvent["event_id"] != latest {
t.Errorf("%s: latest_event = %v, want %s", who, latestEvent["event_id"], latest)
}
if summary["current_user_participated"] != participated {
t.Errorf("%s: current_user_participated = %v, want %v", who, summary["current_user_participated"], participated)
}
unsigned, _ := event["unsigned"].(map[string]any)
relations, _ := unsigned["m.relations"].(map[string]any)
replace, _ := relations["m.replace"].(map[string]any)
if replace["event_id"] != edit {
t.Errorf("%s: m.replace = %v, want alice's own edit %s", who, replace["event_id"], edit)
}
}
// Alice started the thread and Bob replied in it: both took part.
check("alice", alice, true)
check("bob", bob, true)
carol := ts.register("carol", "correct-horse-battery-staple")
expectStatus(t, ts.post("/_matrix/client/v3/join/"+escaped, carol.AccessToken, map[string]any{}), http.StatusOK)
check("carol", carol, false)
// The same summary rides along on /messages.
page := ts.get("/_matrix/client/v3/rooms/"+escaped+"/messages?dir=b&limit=50", alice.AccessToken)
expectStatus(t, page, http.StatusOK)
chunk, _ := jsonBody(t, page)["chunk"].([]any)
found := false
for _, raw := range chunk {
e, _ := raw.(map[string]any)
if e["event_id"] == root {
found = threadSummaryOf(t, e) != nil
}
}
if !found {
t.Error("the root in /messages carries no m.thread summary")
}
}
func TestThreadsEndpoint(t *testing.T) {
ts := newTestServer(t, nil)
alice := ts.register("alice", "correct-horse-battery-staple")
bob := ts.register("bob", "correct-horse-battery-staple")
roomID := ts.createRoom(alice, map[string]any{"preset": "public_chat"})
escaped := url.PathEscape(roomID)
expectStatus(t, ts.post("/_matrix/client/v3/join/"+escaped, bob.AccessToken, map[string]any{}), http.StatusOK)
first := ts.sendMessage(alice, roomID, "first root")
second := ts.sendMessage(bob, roomID, "second root")
ts.sendMessage(alice, roomID, "not a thread")
ts.threadReply(alice, roomID, first, "in the first")
ts.threadReply(alice, roomID, second, "in the second")
ts.threadReply(alice, roomID, first, "the first again")
threads := func(acc account, query string) relationsResult {
t.Helper()
rec := ts.get("/_matrix/client/v1/rooms/"+escaped+"/threads"+query, acc.AccessToken)
expectStatus(t, rec, http.StatusOK)
return decode[relationsResult](t, rec)
}
all := threads(bob, "")
if got := idsOf(all.Chunk); !equalStrings(got, []string{first, second}) {
t.Fatalf("threads = %v, want the first (latest activity) then the second", got)
}
// Bob started the second thread and never replied in the first.
mine := threads(bob, "?include=participated")
if got := idsOf(mine.Chunk); !equalStrings(got, []string{second}) {
t.Errorf("participated threads = %v, want only bob's own", got)
}
page1 := threads(bob, "?limit=1")
if got := idsOf(page1.Chunk); !equalStrings(got, []string{first}) || page1.NextBatch == "" {
t.Fatalf("first page = %v, next_batch %q", got, page1.NextBatch)
}
page2 := threads(bob, "?limit=1&from="+url.QueryEscape(page1.NextBatch))
if got := idsOf(page2.Chunk); !equalStrings(got, []string{second}) || page2.NextBatch != "" {
t.Errorf("second page = %v, next_batch %q", got, page2.NextBatch)
}
rec := ts.get("/_matrix/client/v1/rooms/"+escaped+"/threads?include=everything", bob.AccessToken)
expectStatus(t, rec, http.StatusBadRequest)
}
func TestRedactedRelationDisappears(t *testing.T) {
ts := newTestServer(t, nil)
alice := ts.register("alice", "correct-horse-battery-staple")
roomID := ts.createRoom(alice, nil)
escaped := url.PathEscape(roomID)
root := ts.sendMessage(alice, roomID, "root")
reply := ts.threadReply(alice, roomID, root, "reply")
expectStatus(t, ts.put("/_matrix/client/v3/rooms/"+escaped+"/redact/"+url.PathEscape(reply)+"/r1",
alice.AccessToken, map[string]any{}), http.StatusOK)
if got := ts.relations(alice, roomID, url.PathEscape(root), nil); len(got.Chunk) != 0 {
t.Errorf("relations after the reply was redacted = %v", idsOf(got.Chunk))
}
rec := ts.get("/_matrix/client/v3/rooms/"+escaped+"/event/"+url.PathEscape(root), alice.AccessToken)
if summary := threadSummaryOf(t, jsonBody(t, rec)); summary != nil {
t.Errorf("the root still has a thread summary: %v", summary)
}
}
func TestRelationsOfAnInvisibleEvent(t *testing.T) {
ts := newTestServer(t, nil)
alice := ts.register("alice", "correct-horse-battery-staple")
bob := ts.register("bob", "correct-horse-battery-staple")
roomID := ts.createRoom(alice, nil)
root := ts.sendMessage(alice, roomID, "secret")
rec := ts.get("/_matrix/client/v1/rooms/"+url.PathEscape(roomID)+"/relations/"+url.PathEscape(root),
bob.AccessToken)
expectStatus(t, rec, http.StatusForbidden)
}
func idsOf(events []syncEvent) []string {
out := make([]string, 0, len(events))
for _, e := range events {
out = append(out, e.EventID)
}
return out
}
func equalStrings(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
+27
View File
@@ -0,0 +1,27 @@
package clientapi
import (
"net/http"
"github.com/dariuszjaniak/warp/server/internal/httputil"
"github.com/dariuszjaniak/warp/server/internal/syncapi"
"github.com/dariuszjaniak/warp/server/internal/userapi"
)
// handleSearch implements POST /_matrix/client/v3/search.
func (c *ClientAPI) handleSearch(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
var body syncapi.SearchBody
if err := httputil.DecodeJSON(w, r, &body); err != nil {
return err
}
resp, err := c.sync.Search(r.Context(), syncapi.SearchRequest{
User: auth.UserID,
NextBatch: r.URL.Query().Get("next_batch"),
Body: body,
})
if err != nil {
return syncError(err)
}
httputil.WriteJSON(w, http.StatusOK, resp)
return nil
}
+147
View File
@@ -0,0 +1,147 @@
package clientapi_test
import (
"net/http"
"net/url"
"testing"
)
type searchResult struct {
SearchCategories struct {
RoomEvents struct {
Count int `json:"count"`
Highlights []string `json:"highlights"`
NextBatch string `json:"next_batch"`
Results []struct {
Rank float64 `json:"rank"`
Result syncEvent `json:"result"`
Context *struct {
EventsBefore []syncEvent `json:"events_before"`
EventsAfter []syncEvent `json:"events_after"`
ProfileInfo map[string]map[string]any `json:"profile_info"`
} `json:"context"`
} `json:"results"`
State map[string][]syncEvent `json:"state"`
Groups map[string]map[string]map[string]any `json:"groups"`
} `json:"room_events"`
} `json:"search_categories"`
}
func (ts *testServer) search(acc account, criteria map[string]any, nextBatch string) searchResult {
ts.t.Helper()
path := "/_matrix/client/v3/search"
if nextBatch != "" {
path += "?next_batch=" + url.QueryEscape(nextBatch)
}
rec := ts.post(path, acc.AccessToken, map[string]any{
"search_categories": map[string]any{"room_events": criteria},
})
expectStatus(ts.t, rec, http.StatusOK)
return decode[searchResult](ts.t, rec)
}
func TestSearchFindsOnlyWhatTheUserMaySee(t *testing.T) {
ts := newTestServer(t, nil)
alice := ts.register("alice", "correct-horse-battery-staple")
bob := ts.register("bob", "correct-horse-battery-staple")
shared := ts.createRoom(alice, map[string]any{"preset": "public_chat"})
private := ts.createRoom(alice, nil)
expectStatus(t, ts.post("/_matrix/client/v3/join/"+url.PathEscape(shared), bob.AccessToken, map[string]any{}), http.StatusOK)
ts.sendMessage(alice, shared, "the quick brown fox")
ts.sendMessage(alice, private, "a secret fox")
ts.sendMessage(bob, shared, "lazy dog")
got := ts.search(bob, map[string]any{"search_term": "fox"}, "").SearchCategories.RoomEvents
if got.Count != 1 || len(got.Results) != 1 || got.Results[0].Result.Content["body"] != "the quick brown fox" {
t.Fatalf("bob's search for fox = %+v, want the one message in the room he is in", got)
}
if len(got.Highlights) != 1 || got.Highlights[0] != "fox" {
t.Errorf("highlights = %v", got.Highlights)
}
all := ts.search(alice, map[string]any{"search_term": "fox"}, "").SearchCategories.RoomEvents
if all.Count != 2 {
t.Errorf("alice's search for fox found %d, want both rooms' messages", all.Count)
}
only := ts.search(alice, map[string]any{
"search_term": "fox",
"filter": map[string]any{"rooms": []string{private}},
}, "").SearchCategories.RoomEvents
if only.Count != 1 || only.Results[0].Result.RoomID != private {
t.Errorf("search filtered to one room = %+v", only)
}
// After Bob leaves, what was said while he was there stays findable
// and what comes later does not.
expectStatus(t, ts.post("/_matrix/client/v3/rooms/"+url.PathEscape(shared)+"/leave", bob.AccessToken, map[string]any{}), http.StatusOK)
ts.sendMessage(alice, shared, "another fox, later")
after := ts.search(bob, map[string]any{"search_term": "fox"}, "").SearchCategories.RoomEvents
if after.Count != 1 {
t.Errorf("after leaving, bob's search found %d, want only the message from before", after.Count)
}
}
func TestSearchContextStateGroupsAndPaging(t *testing.T) {
ts := newTestServer(t, nil)
alice := ts.register("alice", "correct-horse-battery-staple")
roomID := ts.createRoom(alice, nil)
ts.sendMessage(alice, roomID, "before")
ts.sendMessage(alice, roomID, "needle one")
ts.sendMessage(alice, roomID, "after")
ts.sendMessage(alice, roomID, "needle two")
ts.sendMessage(alice, roomID, "needle three")
got := ts.search(alice, map[string]any{
"search_term": "needle",
"order_by": "recent",
"event_context": map[string]any{"before_limit": 1, "after_limit": 1, "include_profile": true},
"include_state": true,
"groupings": map[string]any{"group_by": []map[string]any{{"key": "room_id"}}},
"filter": map[string]any{"limit": 2},
}, "").SearchCategories.RoomEvents
if got.Count != 3 || len(got.Results) != 2 {
t.Fatalf("first page: count %d, %d results", got.Count, len(got.Results))
}
if body := got.Results[0].Result.Content["body"]; body != "needle three" {
t.Errorf("newest first: got %v", body)
}
ctx := got.Results[1].Context
if ctx == nil || len(ctx.EventsBefore) != 1 || len(ctx.EventsAfter) != 1 ||
ctx.EventsBefore[0].Content["body"] != "after" || ctx.EventsAfter[0].Content["body"] != "needle three" {
t.Errorf("context of needle two = %+v", ctx)
}
if _, ok := ctx.ProfileInfo[alice.UserID]; !ok {
t.Errorf("profile_info has no entry for alice: %v", ctx.ProfileInfo)
}
if len(got.State[roomID]) == 0 {
t.Error("include_state returned no state for the room")
}
if group := got.Groups["room_id"][roomID]; group == nil {
t.Errorf("groups = %v", got.Groups)
}
if got.NextBatch == "" {
t.Fatal("a full first page carried no next_batch")
}
next := ts.search(alice, map[string]any{
"search_term": "needle", "order_by": "recent", "filter": map[string]any{"limit": 2},
}, got.NextBatch).SearchCategories.RoomEvents
if len(next.Results) != 1 || next.Results[0].Result.Content["body"] != "needle one" {
t.Errorf("second page = %+v", next.Results)
}
ranked := ts.search(alice, map[string]any{"search_term": "needle", "filter": map[string]any{"limit": 2}}, "").SearchCategories.RoomEvents
rest := ts.search(alice, map[string]any{"search_term": "needle", "filter": map[string]any{"limit": 2}}, ranked.NextBatch).SearchCategories.RoomEvents
if len(ranked.Results)+len(rest.Results) != 3 || rest.NextBatch != "" {
t.Errorf("ranked pages: %d then %d results, next %q", len(ranked.Results), len(rest.Results), rest.NextBatch)
}
rec := ts.post("/_matrix/client/v3/search", alice.AccessToken, map[string]any{
"search_categories": map[string]any{"room_events": map[string]any{"search_term": "x", "order_by": "loudest"}},
})
expectStatus(t, rec, http.StatusBadRequest)
}
+47
View File
@@ -0,0 +1,47 @@
package clientapi
import (
"errors"
"net/http"
"github.com/dariuszjaniak/warp/server/internal/httputil"
"github.com/dariuszjaniak/warp/server/internal/syncapi"
"github.com/dariuszjaniak/warp/server/internal/userapi"
)
// slidingSyncPath is MSC4186's endpoint. It is still unstable: the MSC has
// not been merged, and the prefix is what every client implementing it
// sends.
const slidingSyncPath = "/_matrix/client/unstable/org.matrix.simplified_msc3575/sync"
// handleSlidingSync implements POST slidingSyncPath.
func (c *ClientAPI) handleSlidingSync(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
query := r.URL.Query()
timeout, err := durationParam(query.Get("timeout"))
if err != nil {
return err
}
var body syncapi.SlidingBody
if _, err := httputil.DecodeOptionalJSON(w, r, &body); err != nil {
return err
}
resp, err := c.sync.SlidingSync(r.Context(), syncapi.SlidingRequest{
User: auth.UserID,
Localpart: auth.Localpart,
Device: auth.DeviceID,
Pos: query.Get("pos"),
Timeout: timeout,
SetPresence: query.Get("set_presence"),
Body: body,
})
if err != nil {
if ctxErr := r.Context().Err(); ctxErr != nil && errors.Is(err, ctxErr) {
return nil //nolint:nilerr // the client is gone; there is nobody to answer
}
return syncError(err)
}
httputil.NoCache(w)
httputil.WriteJSON(w, http.StatusOK, resp)
return nil
}
+332
View File
@@ -0,0 +1,332 @@
package clientapi_test
import (
"fmt"
"net/http"
"net/url"
"testing"
"time"
"github.com/dariuszjaniak/warp/server/internal/httputil"
)
// Simplified Sliding Sync (MSC4186).
const slidingPath = "/_matrix/client/unstable/org.matrix.simplified_msc3575/sync"
type slidingResult struct {
Pos string `json:"pos"`
Lists map[string]struct {
Count int `json:"count"`
} `json:"lists"`
Rooms map[string]struct {
Name string `json:"name"`
Initial bool `json:"initial"`
RequiredState []syncEvent `json:"required_state"`
Timeline []syncEvent `json:"timeline"`
PrevBatch string `json:"prev_batch"`
Limited bool `json:"limited"`
NumLive *int `json:"num_live"`
BumpStamp int64 `json:"bump_stamp"`
JoinedCount *int `json:"joined_count"`
InviteState []map[string]any `json:"invite_state"`
Heroes []map[string]any `json:"heroes"`
} `json:"rooms"`
Extensions struct {
ToDevice *struct {
NextBatch string `json:"next_batch"`
Events []map[string]any `json:"events"`
} `json:"to_device"`
E2EE *struct {
DeviceLists struct {
Changed []string `json:"changed"`
} `json:"device_lists"`
DeviceOneTimeKeysCount map[string]int `json:"device_one_time_keys_count"`
} `json:"e2ee"`
AccountData *struct {
Global []map[string]any `json:"global"`
Rooms map[string][]map[string]any `json:"rooms"`
} `json:"account_data"`
Receipts *struct {
Rooms map[string]map[string]any `json:"rooms"`
} `json:"receipts"`
Typing *struct {
Rooms map[string]map[string]any `json:"rooms"`
} `json:"typing"`
} `json:"extensions"`
}
func (ts *testServer) slidingRaw(acc account, pos string, timeout time.Duration, body map[string]any) (int, slidingResult, map[string]any) {
ts.t.Helper()
query := url.Values{}
if pos != "" {
query.Set("pos", pos)
}
query.Set("timeout", fmt.Sprint(timeout.Milliseconds()))
rec := ts.post(slidingPath+"?"+query.Encode(), acc.AccessToken, body)
if rec.Code != http.StatusOK {
return rec.Code, slidingResult{}, jsonBody(ts.t, rec)
}
return rec.Code, decode[slidingResult](ts.t, rec), nil
}
func (ts *testServer) sliding(acc account, pos string, body map[string]any) slidingResult {
ts.t.Helper()
code, result, errBody := ts.slidingRaw(acc, pos, 0, body)
if code != http.StatusOK {
ts.t.Fatalf("sliding sync answered %d: %v", code, errBody)
}
if result.Pos == "" {
ts.t.Fatal("sliding sync returned no pos")
}
return result
}
func roomList(ranges [][2]int, timelineLimit int, requiredState [][]string) map[string]any {
return map[string]any{
"lists": map[string]any{
"all": map[string]any{
"ranges": ranges,
"timeline_limit": timelineLimit,
"required_state": requiredState,
},
},
}
}
func TestSlidingSyncListWindowAndIncrementals(t *testing.T) {
ts := newTestServer(t, nil)
alice := ts.register("alice", "correct-horse-battery-staple")
oldest := ts.createRoom(alice, map[string]any{"name": "Oldest"})
middle := ts.createRoom(alice, map[string]any{"name": "Middle"})
newest := ts.createRoom(alice, map[string]any{"name": "Newest"})
ts.sendMessage(alice, oldest, "o1")
ts.sendMessage(alice, middle, "m1")
ts.sendMessage(alice, newest, "n1")
ts.sendMessage(alice, newest, "n2")
body := roomList([][2]int{{0, 1}}, 2, [][]string{{"m.room.name", ""}})
first := ts.sliding(alice, "", body)
if first.Lists["all"].Count != 3 {
t.Fatalf("list count = %d, want 3", first.Lists["all"].Count)
}
if len(first.Rooms) != 2 {
t.Fatalf("rooms = %v, want the two most recently active", keysOf(first.Rooms))
}
room, ok := first.Rooms[newest]
if !ok || !room.Initial || room.Name != "Newest" {
t.Fatalf("newest room = %+v", room)
}
if got := bodiesOf(room.Timeline); len(got) != 2 || got[1] != "n2" || !room.Limited || room.PrevBatch == "" {
t.Errorf("newest timeline = %v (limited %v, prev_batch %q)", got, room.Limited, room.PrevBatch)
}
if len(room.RequiredState) != 1 || room.RequiredState[0].Type != "m.room.name" {
t.Errorf("required_state = %+v", room.RequiredState)
}
if room.JoinedCount == nil || *room.JoinedCount != 1 || room.BumpStamp == 0 {
t.Errorf("counts and bump stamp = %v %d", room.JoinedCount, room.BumpStamp)
}
if _, ok := first.Rooms[middle]; !ok {
t.Errorf("middle room missing from the window: %v", keysOf(first.Rooms))
}
// Nothing has happened: nothing is sent.
idle := ts.sliding(alice, first.Pos, body)
if len(idle.Rooms) != 0 {
t.Errorf("an idle incremental sent %v", keysOf(idle.Rooms))
}
// A message in a room already sent comes as an increment.
ts.sendMessage(alice, middle, "m2")
inc := ts.sliding(alice, idle.Pos, body)
got, ok := inc.Rooms[middle]
if !ok || got.Initial || len(got.Timeline) != 1 || got.Timeline[0].Content["body"] != "m2" {
t.Fatalf("increment for middle = %+v (rooms %v)", got, keysOf(inc.Rooms))
}
if got.NumLive == nil || *got.NumLive != 1 {
t.Errorf("num_live = %v, want 1", got.NumLive)
}
// The oldest room becomes active, moves into the window, and is sent
// in full because this connection has never seen it.
ts.sendMessage(alice, oldest, "o2")
bumped := ts.sliding(alice, inc.Pos, body)
if o, ok := bumped.Rooms[oldest]; !ok || !o.Initial {
t.Errorf("the bumped room = %+v (rooms %v)", o, keysOf(bumped.Rooms))
}
// A retry of an earlier pos still works; a made-up one does not.
if retry := ts.sliding(alice, inc.Pos, body); retry.Pos == "" {
t.Error("retrying the previous pos failed")
}
code, _, errBody := ts.slidingRaw(alice, "nonsense.5", 0, body)
if code != http.StatusBadRequest || errBody["errcode"] != httputil.ErrCodeUnknownPos {
t.Errorf("an unknown pos answered %d %v, want M_UNKNOWN_POS", code, errBody)
}
}
func TestSlidingSyncInvitesSubscriptionsAndLeaves(t *testing.T) {
ts := newTestServer(t, nil)
alice := ts.register("alice", "correct-horse-battery-staple")
bob := ts.register("bob", "correct-horse-battery-staple")
joined := ts.createRoom(bob, nil)
ts.sendMessage(bob, joined, "hello")
invited := ts.createRoom(alice, map[string]any{"name": "Party", "invite": []string{bob.UserID}})
body := map[string]any{
"lists": map[string]any{
"invites": map[string]any{"ranges": [][2]int{{0, 10}}, "filters": map[string]any{"is_invite": true}},
"rooms": map[string]any{
"ranges": [][2]int{{0, 10}}, "filters": map[string]any{"is_invite": false},
"timeline_limit": 5, "required_state": [][]string{{"m.room.member", "$LAZY"}},
},
},
}
first := ts.sliding(bob, "", body)
if first.Lists["invites"].Count != 1 || first.Lists["rooms"].Count != 1 {
t.Fatalf("list counts = %+v", first.Lists)
}
if inv := first.Rooms[invited]; len(inv.InviteState) == 0 || inv.Timeline != nil {
t.Errorf("the invite = %+v, want stripped state and no timeline", inv)
}
lazy := first.Rooms[joined].RequiredState
if len(lazy) != 1 || lazy[0].Type != "m.room.member" || *lazy[0].StateKey != bob.UserID {
t.Errorf("lazy-loaded members = %+v, want only bob, who sent the timeline", lazy)
}
// A subscription to a room outside every list.
public := ts.createRoom(alice, map[string]any{"name": "Public", "preset": "public_chat"})
expectStatus(t, ts.post("/_matrix/client/v3/join/"+url.PathEscape(public), bob.AccessToken, map[string]any{}), http.StatusOK)
sub := ts.sliding(bob, first.Pos, map[string]any{
"room_subscriptions": map[string]any{public: map[string]any{"timeline_limit": 1, "required_state": [][]string{{"*", "*"}}}},
})
if r, ok := sub.Rooms[public]; !ok || !r.Initial || len(r.RequiredState) < 5 {
t.Fatalf("subscribed room = %+v", r)
}
// Joining the invited room turns it into a room sent in full.
expectStatus(t, ts.post("/_matrix/client/v3/join/"+url.PathEscape(invited), bob.AccessToken, map[string]any{}), http.StatusOK)
afterJoin := ts.sliding(bob, sub.Pos, body)
if r, ok := afterJoin.Rooms[invited]; !ok || !r.Initial || len(r.Timeline) == 0 {
t.Errorf("the room just joined = %+v", r)
}
// Leaving is reported, with the leave in the timeline.
expectStatus(t, ts.post("/_matrix/client/v3/rooms/"+url.PathEscape(invited)+"/leave", bob.AccessToken, map[string]any{}), http.StatusOK)
afterLeave := ts.sliding(bob, afterJoin.Pos, body)
r, ok := afterLeave.Rooms[invited]
if !ok || len(r.Timeline) == 0 || r.Timeline[len(r.Timeline)-1].Type != "m.room.member" {
t.Errorf("the room just left = %+v (rooms %v)", r, keysOf(afterLeave.Rooms))
}
}
func TestSlidingSyncExtensions(t *testing.T) {
ts := newTestServer(t, nil)
alice := ts.register("alice", "correct-horse-battery-staple")
roomID := ts.createRoom(alice, nil)
eventID := ts.sendMessage(alice, roomID, "read me")
body := roomList([][2]int{{0, 0}}, 1, nil)
body["extensions"] = map[string]any{
"to_device": map[string]any{"enabled": true},
"e2ee": map[string]any{"enabled": true},
"account_data": map[string]any{"enabled": true},
"receipts": map[string]any{"enabled": true},
"typing": map[string]any{"enabled": true},
}
expectStatus(t, ts.put("/_matrix/client/v3/user/"+url.PathEscape(alice.UserID)+"/account_data/com.example.setting",
alice.AccessToken, map[string]any{"on": true}), http.StatusOK)
expectStatus(t, ts.post(fmt.Sprintf("/_matrix/client/v3/rooms/%s/receipt/m.read/%s",
url.PathEscape(roomID), url.PathEscape(eventID)), alice.AccessToken, map[string]any{}), http.StatusOK)
expectStatus(t, ts.put("/_matrix/client/v3/sendToDevice/m.test/t1", alice.AccessToken, map[string]any{
"messages": map[string]any{alice.UserID: map[string]any{alice.DeviceID: map[string]any{"hello": "device"}}},
}), http.StatusOK)
first := ts.sliding(alice, "", body)
ext := first.Extensions
if ext.ToDevice == nil || len(ext.ToDevice.Events) != 1 || ext.ToDevice.NextBatch == "" {
t.Fatalf("to_device = %+v", ext.ToDevice)
}
if ext.E2EE == nil || ext.E2EE.DeviceOneTimeKeysCount == nil {
t.Errorf("e2ee = %+v", ext.E2EE)
}
foundSetting := false
if ext.AccountData != nil {
for _, e := range ext.AccountData.Global {
if e["type"] == "com.example.setting" {
foundSetting = true
}
}
}
if !foundSetting {
t.Errorf("account_data = %+v", ext.AccountData)
}
if ext.Receipts == nil || ext.Receipts.Rooms[roomID] == nil {
t.Errorf("receipts = %+v", ext.Receipts)
}
// Coming back with the to-device token acknowledges the batch.
body["extensions"].(map[string]any)["to_device"] = map[string]any{"enabled": true, "since": ext.ToDevice.NextBatch}
second := ts.sliding(alice, first.Pos, body)
if second.Extensions.ToDevice == nil || len(second.Extensions.ToDevice.Events) != 0 {
t.Errorf("an acknowledged batch came again: %+v", second.Extensions.ToDevice)
}
// Typing is sent when it starts and again when it stops.
expectStatus(t, ts.put(fmt.Sprintf("/_matrix/client/v3/rooms/%s/typing/%s", url.PathEscape(roomID), url.PathEscape(alice.UserID)),
alice.AccessToken, map[string]any{"typing": true, "timeout": 30000}), http.StatusOK)
typing := ts.sliding(alice, second.Pos, body)
if typing.Extensions.Typing == nil || typing.Extensions.Typing.Rooms[roomID] == nil {
t.Fatalf("typing = %+v", typing.Extensions.Typing)
}
again := ts.sliding(alice, typing.Pos, body)
if len(again.Extensions.Typing.Rooms) != 0 {
t.Errorf("unchanged typing was sent again: %+v", again.Extensions.Typing.Rooms)
}
expectStatus(t, ts.put(fmt.Sprintf("/_matrix/client/v3/rooms/%s/typing/%s", url.PathEscape(roomID), url.PathEscape(alice.UserID)),
alice.AccessToken, map[string]any{"typing": false}), http.StatusOK)
stopped := ts.sliding(alice, again.Pos, body)
if stopped.Extensions.Typing.Rooms[roomID] == nil {
t.Errorf("stopping typing was not sent: %+v", stopped.Extensions.Typing.Rooms)
}
}
func TestSlidingSyncLongPoll(t *testing.T) {
ts := newTestServer(t, nil)
alice := ts.register("alice", "correct-horse-battery-staple")
roomID := ts.createRoom(alice, nil)
body := roomList([][2]int{{0, 0}}, 5, nil)
first := ts.sliding(alice, "", body)
done := make(chan slidingResult, 1)
start := time.Now()
go func() {
_, result, _ := ts.slidingRaw(alice, first.Pos, 10*time.Second, body)
done <- result
}()
time.Sleep(200 * time.Millisecond)
ts.sendMessage(alice, roomID, "wake up")
select {
case result := <-done:
if r, ok := result.Rooms[roomID]; !ok || len(r.Timeline) != 1 {
t.Errorf("the woken response = %+v", result.Rooms)
}
if time.Since(start) > 5*time.Second {
t.Errorf("the long poll took %s to wake", time.Since(start))
}
case <-time.After(15 * time.Second):
t.Fatal("the long poll never returned")
}
}
func keysOf[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
return out
}
+105
View File
@@ -0,0 +1,105 @@
package clientapi
import (
"net/http"
"strconv"
"strings"
"github.com/dariuszjaniak/warp/server/internal/httputil"
"github.com/dariuszjaniak/warp/server/internal/matrix/id"
"github.com/dariuszjaniak/warp/server/internal/roomserver"
"github.com/dariuszjaniak/warp/server/internal/userapi"
)
// hierarchyResponse is what GET /hierarchy returns.
type hierarchyResponse struct {
Rooms []roomserver.SpaceRoom `json:"rooms"`
NextBatch string `json:"next_batch,omitempty"`
}
// hierarchyTokenPrefix marks a /hierarchy token: how far into the walk
// the next page starts.
const hierarchyTokenPrefix = "h"
// handleHierarchy implements GET /_matrix/client/v1/rooms/{roomID}/hierarchy.
func (c *ClientAPI) handleHierarchy(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
roomID, err := pathRoomID(r)
if err != nil {
return err
}
query := r.URL.Query()
limit, err := intParam(query.Get("limit"))
if err != nil {
return err
}
suggestedOnly, err := boolParam(query.Get("suggested_only"))
if err != nil {
return err
}
maxDepth := -1
if raw := query.Get("max_depth"); raw != "" {
if maxDepth, err = strconv.Atoi(raw); err != nil || maxDepth < 0 {
return httputil.InvalidParam("max_depth must be a non-negative integer")
}
}
from := 0
if raw := query.Get("from"); raw != "" {
body, ok := strings.CutPrefix(raw, hierarchyTokenPrefix)
if from, err = strconv.Atoi(body); !ok || err != nil || from < 0 {
return httputil.InvalidParam("%q is not a hierarchy token", raw)
}
}
page, err := c.rooms.Hierarchy(r.Context(), roomserver.HierarchyRequest{
RoomID: roomID,
User: auth.UserID,
SuggestedOnly: suggestedOnly,
MaxDepth: maxDepth,
From: from,
Limit: limit,
})
if err != nil {
return roomError(err)
}
resp := hierarchyResponse{Rooms: page.Rooms}
if resp.Rooms == nil {
resp.Rooms = []roomserver.SpaceRoom{}
}
if page.Next != 0 {
resp.NextBatch = hierarchyTokenPrefix + strconv.Itoa(page.Next)
}
httputil.WriteJSON(w, http.StatusOK, resp)
return nil
}
// handleRoomSummary implements
// GET /_matrix/client/v1/room_summary/{roomIDOrAlias}.
func (c *ClientAPI) handleRoomSummary(w http.ResponseWriter, r *http.Request, auth userapi.Authenticated) error {
target := r.PathValue("roomIDOrAlias")
via := r.URL.Query()["via"]
roomID := target
switch {
case strings.HasPrefix(target, "#"):
resolved, err := c.resolveAlias(r.Context(), target)
if err != nil {
return roomError(err)
}
roomID = resolved.RoomID
via = append(via, resolved.Servers...)
case id.ValidateRoomID(target) != nil:
return httputil.InvalidParam("%s is not a room ID or alias", target)
}
resp, err := c.rooms.RoomSummary(r.Context(), roomserver.RoomSummaryRequest{
RoomID: roomID,
User: auth.UserID,
Via: via,
})
if err != nil {
return roomError(err)
}
httputil.WriteJSON(w, http.StatusOK, resp)
return nil
}
+182
View File
@@ -0,0 +1,182 @@
package clientapi_test
import (
"net/http"
"net/url"
"testing"
)
type hierarchyResult struct {
Rooms []struct {
RoomID string `json:"room_id"`
Name string `json:"name"`
RoomType string `json:"room_type"`
JoinRule string `json:"join_rule"`
NumJoinedMembers int `json:"num_joined_members"`
ChildrenState []map[string]any `json:"children_state"`
} `json:"rooms"`
NextBatch string `json:"next_batch"`
}
func (h hierarchyResult) ids() []string {
out := make([]string, 0, len(h.Rooms))
for _, r := range h.Rooms {
out = append(out, r.RoomID)
}
return out
}
func (ts *testServer) hierarchy(acc account, roomID, query string) hierarchyResult {
ts.t.Helper()
rec := ts.get("/_matrix/client/v1/rooms/"+url.PathEscape(roomID)+"/hierarchy"+query, acc.AccessToken)
expectStatus(ts.t, rec, http.StatusOK)
return decode[hierarchyResult](ts.t, rec)
}
// addChild puts a room into a space.
func (ts *testServer) addChild(acc account, spaceID, childID string, content map[string]any) {
ts.t.Helper()
if content == nil {
content = map[string]any{}
}
if _, ok := content["via"]; !ok {
content["via"] = []string{"warp.example"}
}
rec := ts.put("/_matrix/client/v3/rooms/"+url.PathEscape(spaceID)+"/state/m.space.child/"+url.PathEscape(childID),
acc.AccessToken, content)
expectStatus(ts.t, rec, http.StatusOK)
}
func (ts *testServer) createSpace(acc account, name string, public bool) string {
ts.t.Helper()
body := map[string]any{"name": name, "creation_content": map[string]any{"type": "m.space"}}
if public {
body["preset"] = "public_chat"
}
return ts.createRoom(acc, body)
}
func TestSpaceHierarchy(t *testing.T) {
ts := newTestServer(t, nil)
alice := ts.register("alice", "correct-horse-battery-staple")
bob := ts.register("bob", "correct-horse-battery-staple")
space := ts.createSpace(alice, "Space", true)
sub := ts.createSpace(alice, "Subspace", true)
general := ts.createRoom(alice, map[string]any{"name": "General", "preset": "public_chat"})
random := ts.createRoom(alice, map[string]any{"name": "Random", "preset": "public_chat"})
secret := ts.createRoom(alice, map[string]any{"name": "Secret"})
deep := ts.createRoom(alice, map[string]any{"name": "Deep", "preset": "public_chat"})
ts.addChild(alice, space, random, map[string]any{"order": "b"})
ts.addChild(alice, space, general, map[string]any{"order": "a", "suggested": true})
ts.addChild(alice, space, sub, nil)
ts.addChild(alice, space, secret, nil)
ts.addChild(alice, sub, deep, nil)
// A child event with no via is a removed child.
removed := ts.createRoom(alice, map[string]any{"name": "Gone", "preset": "public_chat"})
ts.addChild(alice, space, removed, map[string]any{"via": []string{}})
// Alice is in every room, so she sees all of them: the root, its
// children by order and then by timestamp, then the subspace's child.
all := ts.hierarchy(alice, space, "")
want := []string{space, general, random, sub, secret, deep}
if got := all.ids(); !equalStrings(got, want) {
t.Fatalf("alice's hierarchy = %v, want %v", got, want)
}
if all.Rooms[0].RoomType != "m.space" || len(all.Rooms[0].ChildrenState) != 4 {
t.Errorf("root = %+v, want a space with four live children", all.Rooms[0])
}
// Bob is in nothing: the invite-only room is hidden from him.
if got := ts.hierarchy(bob, space, "").ids(); !equalStrings(got, []string{space, general, random, sub, deep}) {
t.Errorf("bob's hierarchy = %v", got)
}
if got := ts.hierarchy(alice, space, "?max_depth=1").ids(); !equalStrings(got, []string{space, general, random, sub, secret}) {
t.Errorf("max_depth=1 = %v", got)
}
if got := ts.hierarchy(alice, space, "?suggested_only=true").ids(); !equalStrings(got, []string{space, general}) {
t.Errorf("suggested_only = %v", got)
}
// Paging two at a time walks the same order.
var paged []string
from := ""
for range 10 {
query := "?limit=2"
if from != "" {
query += "&from=" + url.QueryEscape(from)
}
page := ts.hierarchy(alice, space, query)
paged = append(paged, page.ids()...)
if page.NextBatch == "" {
break
}
from = page.NextBatch
}
if !equalStrings(paged, want) {
t.Errorf("paged hierarchy = %v, want %v", paged, want)
}
}
func TestSpaceHierarchyAccess(t *testing.T) {
ts := newTestServer(t, nil)
alice := ts.register("alice", "correct-horse-battery-staple")
bob := ts.register("bob", "correct-horse-battery-staple")
carol := ts.register("carol", "correct-horse-battery-staple")
private := ts.createSpace(alice, "Private", false)
rec := ts.get("/_matrix/client/v1/rooms/"+url.PathEscape(private)+"/hierarchy", bob.AccessToken)
expectStatus(t, rec, http.StatusForbidden)
// A restricted room is visible to the members of the room it lets in.
space := ts.createSpace(alice, "Space", true)
restricted := ts.createRoom(alice, map[string]any{
"name": "Members only",
"initial_state": []map[string]any{{
"type": "m.room.join_rules", "state_key": "",
"content": map[string]any{
"join_rule": "restricted",
"allow": []map[string]any{{"type": "m.room_membership", "room_id": space}},
},
}},
})
ts.addChild(alice, space, restricted, nil)
expectStatus(t, ts.post("/_matrix/client/v3/join/"+url.PathEscape(space), bob.AccessToken, map[string]any{}), http.StatusOK)
if got := ts.hierarchy(bob, space, "").ids(); !equalStrings(got, []string{space, restricted}) {
t.Errorf("a member of the space sees %v, want the restricted room too", got)
}
if got := ts.hierarchy(carol, space, "").ids(); !equalStrings(got, []string{space}) {
t.Errorf("a stranger sees %v, want the space alone", got)
}
}
func TestRoomSummary(t *testing.T) {
ts := newTestServer(t, nil)
alice := ts.register("alice", "correct-horse-battery-staple")
bob := ts.register("bob", "correct-horse-battery-staple")
public := ts.createRoom(alice, map[string]any{
"name": "Lobby", "topic": "Say hello", "preset": "public_chat", "room_alias_name": "lobby",
})
private := ts.createRoom(alice, map[string]any{"name": "Private"})
for _, target := range []string{public, "#lobby:warp.example"} {
rec := ts.get("/_matrix/client/v1/room_summary/"+url.PathEscape(target), bob.AccessToken)
expectStatus(t, rec, http.StatusOK)
body := jsonBody(t, rec)
if body["room_id"] != public || body["name"] != "Lobby" || body["topic"] != "Say hello" ||
body["membership"] != "leave" || body["join_rule"] != "public" || body["room_version"] == nil {
t.Errorf("summary of %s = %v", target, body)
}
}
rec := ts.get("/_matrix/client/v1/room_summary/"+url.PathEscape(private), bob.AccessToken)
expectStatus(t, rec, http.StatusForbidden)
rec = ts.get("/_matrix/client/v1/room_summary/"+url.PathEscape(private), alice.AccessToken)
expectStatus(t, rec, http.StatusOK)
if jsonBody(t, rec)["membership"] != "join" {
t.Errorf("alice's summary of her own room says membership %v", jsonBody(t, rec)["membership"])
}
}
+2
View File
@@ -34,6 +34,8 @@ func syncError(err error) error {
return httputil.InvalidParam("%s", err.Error())
case errors.Is(err, syncapi.ErrInvalidToken):
return httputil.InvalidParam("%s", err.Error())
case errors.Is(err, syncapi.ErrUnknownPos):
return httputil.NewError(http.StatusBadRequest, httputil.ErrCodeUnknownPos, "%s", err.Error())
case errors.Is(err, syncapi.ErrPresenceDisabled):
return httputil.Forbidden("Presence is disabled on this server")
+76
View File
@@ -11,6 +11,8 @@ import (
"path/filepath"
"strings"
"time"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// Config is the whole server configuration.
@@ -31,6 +33,7 @@ type Config struct {
ServerNotices ServerNotices `yaml:"server_notices"`
Logging Logging `yaml:"logging"`
Metrics Metrics `yaml:"metrics"`
Tracing Tracing `yaml:"tracing"`
// path is the file this config was read from, if any.
path string
@@ -108,6 +111,48 @@ type Database struct {
// AutoMigrate runs pending migrations on startup. Turn it off to gate
// schema changes behind an explicit `warp migrate`.
AutoMigrate bool `yaml:"auto_migrate" env:"WARP_DB_AUTO_MIGRATE"`
// PocketBase configures what the pocketbase driver adds over sqlite.
PocketBase PocketBase `yaml:"pocketbase"`
}
// PocketBase configures the pocketbase storage driver.
type PocketBase struct {
// S3 puts media content in an S3-compatible bucket through
// PocketBase's filesystem, instead of under the data directory.
S3 PocketBaseS3 `yaml:"s3"`
}
// PocketBaseS3 is an S3-compatible bucket.
type PocketBaseS3 struct {
Enabled bool `yaml:"enabled" env:"WARP_PB_S3_ENABLED"`
Bucket string `yaml:"bucket" env:"WARP_PB_S3_BUCKET"`
Region string `yaml:"region" env:"WARP_PB_S3_REGION"`
Endpoint string `yaml:"endpoint" env:"WARP_PB_S3_ENDPOINT"`
AccessKey string `yaml:"access_key" env:"WARP_PB_S3_ACCESS_KEY"`
Secret string `yaml:"secret" env:"WARP_PB_S3_SECRET"`
ForcePathStyle bool `yaml:"force_path_style" env:"WARP_PB_S3_FORCE_PATH_STYLE"`
}
// Storage is the storage driver configuration this section describes.
func (d Database) Storage() storage.Config {
s3 := d.PocketBase.S3
return storage.Config{
Driver: d.Driver,
ConnectionString: d.ConnectionString,
MaxOpenConns: d.MaxOpenConns,
MaxIdleConns: d.MaxIdleConns,
ConnMaxLifetime: d.ConnMaxLifetime,
S3: storage.S3Config{
Enabled: s3.Enabled,
Bucket: s3.Bucket,
Region: s3.Region,
Endpoint: s3.Endpoint,
AccessKey: s3.AccessKey,
Secret: s3.Secret,
ForcePathStyle: s3.ForcePathStyle,
},
}
}
// ClientAPI configures the Client-Server API surface.
@@ -391,6 +436,22 @@ type Metrics struct {
Path string `yaml:"path" env:"WARP_METRICS_PATH"`
}
// Tracing configures OpenTelemetry tracing.
type Tracing struct {
Enabled bool `yaml:"enabled" env:"WARP_TRACING_ENABLED"`
// Endpoint is the OTLP/HTTP collector, e.g. "http://localhost:4318".
// Empty falls back to the standard OTEL_EXPORTER_OTLP_ENDPOINT.
Endpoint string `yaml:"endpoint" env:"WARP_TRACING_ENDPOINT"`
// SampleRatio is the fraction of new traces kept, from 0 to 1. A
// request that arrives as part of a sampled trace is always kept.
SampleRatio float64 `yaml:"sample_ratio" env:"WARP_TRACING_SAMPLE_RATIO"`
// ServiceName is what the traces are reported as.
ServiceName string `yaml:"service_name" env:"WARP_TRACING_SERVICE_NAME"`
}
// Path returns the file this config was loaded from, or "" for defaults.
func (c *Config) Path() string { return c.path }
@@ -481,6 +542,7 @@ func Default() *Config {
},
Logging: Logging{Level: "info", Format: "text"},
Metrics: Metrics{Enabled: true, Path: "/metrics"},
Tracing: Tracing{SampleRatio: 1, ServiceName: "warp"},
}
}
@@ -571,6 +633,20 @@ func (c *Config) Validate() error {
if c.Metrics.Enabled && !strings.HasPrefix(c.Metrics.Path, "/") {
bad("metrics.path: must start with /")
}
if c.Tracing.SampleRatio < 0 || c.Tracing.SampleRatio > 1 {
bad("tracing.sample_ratio: must be between 0 and 1")
}
if c.Tracing.Endpoint != "" {
if u, err := url.Parse(c.Tracing.Endpoint); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
bad("tracing.endpoint: must be an http or https URL")
}
}
if c.Database.Driver == "pocketbase" && c.Database.PocketBase.S3.Enabled {
s3 := c.Database.PocketBase.S3
if s3.Bucket == "" || s3.Region == "" || s3.Endpoint == "" || s3.AccessKey == "" || s3.Secret == "" {
bad("database.pocketbase.s3: bucket, region, endpoint, access_key and secret are all required")
}
}
if c.ClientAPI.AccessTokenLifetime < 0 {
bad("client_api.access_token_lifetime: must not be negative")
+6
View File
@@ -76,6 +76,12 @@ func setFromString(fv reflect.Value, raw string) error {
return fmt.Errorf("invalid integer %q", raw)
}
fv.SetInt(n)
case reflect.Float32, reflect.Float64:
f, err := strconv.ParseFloat(raw, 64)
if err != nil {
return fmt.Errorf("invalid number %q", raw)
}
fv.SetFloat(f)
case reflect.Slice:
if fv.Type().Elem().Kind() != reflect.String {
return fmt.Errorf("unsupported slice type %s", fv.Type())
+17 -1
View File
@@ -17,6 +17,7 @@ import (
"github.com/dariuszjaniak/warp/server/internal/buildinfo"
"github.com/dariuszjaniak/warp/server/internal/matrix/signing"
"github.com/dariuszjaniak/warp/server/internal/tracing"
)
// Errors an outbound request can fail with before anything is sent.
@@ -170,7 +171,8 @@ func NewClient(opts ClientOptions) *Client {
key: opts.Key,
resolve: opts.Resolve,
http: &http.Client{
Transport: transport,
// Each request is a client span, and carries the trace on.
Transport: tracing.Transport(transport),
// Federation endpoints do not redirect; following one would
// send a signed request somewhere it was not addressed.
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
@@ -551,6 +553,20 @@ func (c *Client) PublicRooms(ctx context.Context, destination string, limit int,
return out, err
}
// Hierarchy asks a server about one room of a space and its children.
func (c *Client) Hierarchy(ctx context.Context, destination, roomID string, suggestedOnly bool) (json.RawMessage, error) {
query := url.Values{}
if suggestedOnly {
query.Set("suggested_only", "true")
}
var out json.RawMessage
err := c.do(ctx, request{
method: http.MethodGet, destination: destination, signed: true,
path: "/_matrix/federation/v1/hierarchy/" + url.PathEscape(roomID), query: query,
}, &out)
return out, err
}
// Version asks a server what software it runs.
func (c *Client) Version(ctx context.Context, destination string) (VersionResponse, error) {
var out VersionResponse
+14
View File
@@ -157,6 +157,7 @@ func (s *Server) RegisterRoutes(r *httputil.Router) {
authed("GET "+v1+"/query/profile", s.handleQueryProfile)
authed("GET "+v1+"/publicRooms", s.handlePublicRoomsGet)
authed("POST "+v1+"/publicRooms", s.handlePublicRoomsPost)
authed("GET "+v1+"/hierarchy/{roomID}", s.handleHierarchy)
s.registerMediaRoutes(authed)
s.registerKeyRoutes(authed)
@@ -668,6 +669,19 @@ func (s *Server) publicRooms(w http.ResponseWriter, r *http.Request, req roomser
return nil
}
// handleHierarchy implements GET /_matrix/federation/v1/hierarchy/{roomID}:
// another server walking one of this server's spaces.
func (s *Server) handleHierarchy(w http.ResponseWriter, r *http.Request, origin string, _ []byte) error {
roomID := r.PathValue("roomID")
suggestedOnly, _ := strconv.ParseBool(r.URL.Query().Get("suggested_only"))
resp, err := s.rooms.FederatedHierarchy(r.Context(), origin, roomID, suggestedOnly)
if err != nil {
return fedError(err)
}
httputil.WriteJSON(w, http.StatusOK, resp)
return nil
}
// String is for logs.
func (s *Server) String() string { return fmt.Sprintf("federation API for %s", s.serverName) }
+4
View File
@@ -49,6 +49,10 @@ const (
ErrCodeConnectionFailed = "M_CONNECTION_FAILED"
ErrCodeConnectionTimeout = "M_CONNECTION_TIMEOUT"
ErrCodeBadStatus = "M_BAD_STATUS"
// Sliding sync, added in Phase 10: a `pos` the server no longer
// remembers, which tells the client to start a new connection.
ErrCodeUnknownPos = "M_UNKNOWN_POS"
)
// MatrixError is the standard Matrix error envelope. Every non-2xx response
+30 -14
View File
@@ -24,19 +24,17 @@ const (
// ReceiptSet is told about a read receipt a local user set, and clears
// their notifications in the room up to the event it names.
//
// Notifications are not tracked per thread (threads are Phase 10), so a
// receipt counts when it is for the room as a whole or for its main
// timeline, and a receipt inside a thread is left alone: clearing the
// room on it would mark the main timeline read when the user only looked
// at one thread. Thread notifications are cleared when the main receipt
// moves past them.
// Notifications are filed by thread, and a receipt reads the part of the
// room it is for: thread_id "main" the main timeline, a thread root's ID
// that thread, and no thread_id the whole room. Reading one thread leaves
// the main timeline and the other threads unread, and a client that does
// not know about threads sends unthreaded receipts and keeps clearing the
// room as a whole, as before threads.
func (a *API) ReceiptSet(ctx context.Context, userID, roomID, receiptType, threadID, eventID string) {
if receiptType != receiptRead && receiptType != receiptReadPrivate {
return
}
if threadID != "" && threadID != threadMain {
return
}
thread := receiptThread(threadID)
pos, err := a.rooms.EventPosition(ctx, roomID, userID, eventID)
if err != nil {
// A receipt for an event this server does not have — or one the
@@ -47,16 +45,28 @@ func (a *API) ReceiptSet(ctx context.Context, userID, roomID, receiptType, threa
}
return
}
if err := a.markRead(ctx, userID, roomID, pos); err != nil {
if err := a.markRead(ctx, userID, roomID, thread, pos); err != nil {
a.log.Warn("clearing notifications on a receipt failed",
"user_id", userID, "room_id", roomID, "error", err)
}
}
// markRead clears a user's notifications in a room up to a position, and
// wakes their syncs if the counts moved.
func (a *API) markRead(ctx context.Context, userID, roomID string, upTo int64) error {
_, moved, err := a.store.MarkNotificationsRead(ctx, userID, roomID, upTo)
// receiptThread is the part of a room a receipt's thread_id reads, as the
// store names it.
func receiptThread(threadID string) string {
switch threadID {
case "":
return storage.AllThreads
case threadMain:
return storage.ThreadMain
}
return threadID
}
// markRead clears a user's notifications in a room, or in one thread of
// it, up to a position, and wakes their syncs if the counts moved.
func (a *API) markRead(ctx context.Context, userID, roomID, thread string, upTo int64) error {
_, moved, err := a.store.MarkNotificationsRead(ctx, userID, roomID, thread, upTo)
if err != nil {
return err
}
@@ -71,6 +81,12 @@ func (a *API) NotificationCounts(ctx context.Context, userID string, roomIDs []s
return a.store.NotificationCounts(ctx, userID, roomIDs)
}
// ThreadNotificationCounts reads a user's unread counts in the given rooms
// per thread, with the main timeline under storage.ThreadMain.
func (a *API) ThreadNotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]map[string]storage.NotificationCounts, error) {
return a.store.ThreadNotificationCounts(ctx, userID, roomIDs)
}
// NotificationCountChanges lists the rooms whose counts for a user moved
// in a window of the notification stream.
func (a *API) NotificationCountChanges(ctx context.Context, userID string, from, to int64) ([]string, error) {
@@ -0,0 +1,22 @@
package pushserver
import (
"testing"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
func TestReceiptThread(t *testing.T) {
for threadID, want := range map[string]string{
// No thread_id: the receipt reads the room as a whole.
"": storage.AllThreads,
// "main" reads the main timeline, which the store files as ''.
"main": storage.ThreadMain,
// Anything else is a thread root, and reads that thread.
"$root:warp.example": "$root:warp.example",
} {
if got := receiptThread(threadID); got != want {
t.Errorf("receiptThread(%q) = %q, want %q", threadID, got, want)
}
}
}
+2 -1
View File
@@ -28,6 +28,7 @@ import (
"github.com/dariuszjaniak/warp/server/internal/pushrules"
"github.com/dariuszjaniak/warp/server/internal/roomserver"
"github.com/dariuszjaniak/warp/server/internal/storage"
"github.com/dariuszjaniak/warp/server/internal/tracing"
)
// Errors callers have to tell apart. The rule-language sentinels are
@@ -174,7 +175,7 @@ func New(opts Options) (*API, error) {
opts.NotificationRetention = DefaultNotificationRetention
}
if opts.HTTPClient == nil {
opts.HTTPClient = &http.Client{Timeout: opts.GatewayTimeout}
opts.HTTPClient = &http.Client{Timeout: opts.GatewayTimeout, Transport: tracing.Transport(nil)}
}
ctx, cancel := context.WithCancel(context.Background())
+3 -1
View File
@@ -104,7 +104,7 @@ func (a *API) process(ctx context.Context, n roomserver.Notification) error {
if target, ok := e.StateKey(); ok && a.isLocal(target) {
switch membership, _ := e.Membership(); membership {
case event.MembershipLeave, event.MembershipBan:
if err := a.markRead(ctx, target, n.RoomID, n.StreamOrdering); err != nil {
if err := a.markRead(ctx, target, n.RoomID, storage.AllThreads, n.StreamOrdering); err != nil {
return err
}
}
@@ -144,6 +144,7 @@ func (a *API) process(ctx context.Context, n roomserver.Notification) error {
var (
notifications []storage.NewNotification
outcomes = map[string]pushrules.Outcome{}
thread = roomserver.ThreadOf(e)
)
now := a.now().UnixMilli()
for _, userID := range candidates {
@@ -179,6 +180,7 @@ func (a *API) process(ctx context.Context, n roomserver.Notification) error {
RoomID: n.RoomID,
EventID: e.EventID(),
EventPos: n.StreamOrdering,
ThreadID: thread,
TimestampMS: now,
Highlight: outcome.Highlight,
Actions: actions,
@@ -346,6 +346,18 @@ func (t transport) QueryDirectory(ctx context.Context, dest, alias string) (fede
return federation.DirectoryResponse{RoomID: resolved.RoomID, Servers: resolved.Servers}, httpError(err)
}
func (t transport) Hierarchy(ctx context.Context, dest, roomID string, suggestedOnly bool) (json.RawMessage, error) {
api, err := t.to(dest)
if err != nil {
return nil, err
}
resp, err := api.FederatedHierarchy(ctx, t.from.name, roomID, suggestedOnly)
if err != nil {
return nil, httpError(err)
}
return json.Marshal(resp)
}
func (t transport) PublicRooms(ctx context.Context, dest string, limit int, since, search string) (json.RawMessage, error) {
api, err := t.to(dest)
if err != nil {
@@ -1052,3 +1064,69 @@ func (m *mesh) setDisabled(disabled bool) {
defer m.mu.Unlock()
m.disabled = disabled
}
func TestFederatedSpaceHierarchy(t *testing.T) {
m := newMesh(t, serverA, serverB)
a, b := m.servers[serverA], m.servers[serverB]
ctx := t.Context()
space, err := a.api.CreateRoom(ctx, roomserver.CreateRoomRequest{
Creator: userA, Preset: roomserver.PresetPublicChat, Name: "Space",
CreationContent: json.RawMessage(`{"type":"m.space"}`),
})
if err != nil {
t.Fatalf("CreateRoom space: %v", err)
}
public := a.publicRoom(t, userA)
private, err := a.api.CreateRoom(ctx, roomserver.CreateRoomRequest{Creator: userA, Name: "Private"})
if err != nil {
t.Fatalf("CreateRoom private: %v", err)
}
for _, child := range []string{public, private.RoomID} {
key := child
if _, err := a.api.SendEvent(ctx, roomserver.SendEventRequest{
RoomID: space.RoomID, Sender: userA, Type: roomserver.TypeSpaceChild, StateKey: &key,
Content: json.RawMessage(`{"via":["` + serverA + `"]}`),
}); err != nil {
t.Fatalf("add child: %v", err)
}
}
a.takeOutbox()
if _, err := b.api.JoinRoom(ctx, roomserver.JoinRequest{
RoomID: space.RoomID, User: userB, ServerNames: []string{serverA},
}); err != nil {
t.Fatalf("remote join: %v", err)
}
if failures := m.flush(); len(failures) > 0 {
t.Fatalf("delivery failures: %v", failures)
}
// B holds the space itself and asks A about the rooms it is not in:
// the public one comes back, the private one is A's to keep.
page, err := b.api.Hierarchy(ctx, roomserver.HierarchyRequest{RoomID: space.RoomID, User: userB, MaxDepth: -1})
if err != nil {
t.Fatalf("Hierarchy: %v", err)
}
var got []string
for _, room := range page.Rooms {
got = append(got, room.RoomID)
}
if len(got) != 2 || got[0] != space.RoomID || got[1] != public {
t.Fatalf("B's walk of the space = %v, want the space and the public room", got)
}
if page.Rooms[1].Name != "Federated" || page.Rooms[1].NumJoinedMembers != 1 {
t.Errorf("the remote child = %+v", page.Rooms[1])
}
answer, err := a.api.FederatedHierarchy(ctx, serverB, space.RoomID, false)
if err != nil {
t.Fatalf("FederatedHierarchy: %v", err)
}
if len(answer.InaccessibleChildren) != 1 || answer.InaccessibleChildren[0] != private.RoomID {
t.Errorf("inaccessible_children = %v, want the private room", answer.InaccessibleChildren)
}
if _, err := a.api.FederatedHierarchy(ctx, serverB, private.RoomID, false); !errors.Is(err, roomserver.ErrNoSuchRoom) {
t.Errorf("FederatedHierarchy of the private room = %v, want ErrNoSuchRoom", err)
}
}
+616
View File
@@ -0,0 +1,616 @@
package roomserver
import (
"cmp"
"context"
"encoding/json"
"fmt"
"slices"
"strings"
"github.com/dariuszjaniak/warp/server/internal/matrix/auth"
"github.com/dariuszjaniak/warp/server/internal/matrix/event"
"github.com/dariuszjaniak/warp/server/internal/matrix/id"
)
// Spaces: a space is a room whose create event says "type": "m.space",
// and whose m.space.child state events name the rooms in it. Nothing about
// that is enforced by the authorisation rules — a child event is a pointer,
// and the room it points at need not know — so what the server provides is
// the walk: /hierarchy reads a space's tree breadth first, summarising each
// room the user is allowed to know about, and asks the space's other
// servers about the rooms this one is not in.
//
// The same walk, one level deep and judged for a server rather than a user,
// answers the federation endpoint other servers walk Warp's spaces with.
// The space event types.
const (
TypeSpaceChild = "m.space.child"
TypeSpaceParent = "m.space.parent"
// RoomTypeSpace is m.room.create's content.type for a space.
RoomTypeSpace = "m.space"
)
// Limits of a hierarchy walk.
const (
// DefaultHierarchyLimit is the page size when the client names none,
// and MaxHierarchyLimit caps the one it names.
DefaultHierarchyLimit = 50
MaxHierarchyLimit = 100
// MaxHierarchyDepth caps max_depth: a space tree deeper than this is
// either a mistake or an attempt to make the server walk forever.
MaxHierarchyDepth = 50
// maxHierarchyRooms caps how many rooms one walk visits, whatever the
// page size: paging re-walks from the root, so the cap is what bounds
// the work of the last page of a large space.
maxHierarchyRooms = 2000
// maxOrderLength is the longest m.space.child order the spec lets
// sort a child; a longer one is ignored, as if absent.
maxOrderLength = 50
)
// SpaceChildEvent is an m.space.child event as a hierarchy carries it: the
// stripped state event plus its timestamp, which children are sorted by.
type SpaceChildEvent struct {
Type string `json:"type"`
StateKey string `json:"state_key"`
Sender string `json:"sender"`
Content json.RawMessage `json:"content"`
OriginServerTS int64 `json:"origin_server_ts"`
}
// SpaceRoom is one room of a hierarchy: the public face of the room, as
// the public room directory shows it, and the space's children when it has
// any.
type SpaceRoom struct {
RoomID string `json:"room_id"`
Name string `json:"name,omitempty"`
Topic string `json:"topic,omitempty"`
AvatarURL string `json:"avatar_url,omitempty"`
CanonicalAlias string `json:"canonical_alias,omitempty"`
NumJoinedMembers int `json:"num_joined_members"`
WorldReadable bool `json:"world_readable"`
GuestCanJoin bool `json:"guest_can_join"`
JoinRule string `json:"join_rule,omitempty"`
RoomType string `json:"room_type,omitempty"`
RoomVersion string `json:"room_version,omitempty"`
Encryption string `json:"encryption,omitempty"`
// ChildrenState is the room's m.space.child events. It is always there
// on the rooms a client is given, empty for a room that is not a
// space, and left out of the children a federation response lists.
ChildrenState []SpaceChildEvent `json:"children_state,omitempty"`
// AllowedRoomIDs is the rooms whose members may join under a
// restricted join rule. Only a federation response carries it: the
// asking server needs it to decide what its own users may see.
AllowedRoomIDs []string `json:"allowed_room_ids,omitempty"`
}
// HierarchyRequest is one page of a space's tree.
type HierarchyRequest struct {
RoomID string
// User is who is walking. Each room is shown only if they may know
// about it.
User string
SuggestedOnly bool
// MaxDepth is how far below the root to go; zero is the root alone.
// Negative means the server's own limit.
MaxDepth int
// From is how many rooms of the walk earlier pages have already
// returned.
From int
Limit int
}
// HierarchyPage is one page of a space's tree.
type HierarchyPage struct {
Rooms []SpaceRoom
// Next is the From of the next page, zero at the end.
Next int
}
// FederationHierarchy is what GET /_matrix/federation/v1/hierarchy
// answers: the room, the children this server can describe, and the ones
// the asking server may not see.
type FederationHierarchy struct {
Room SpaceRoom `json:"room"`
Children []SpaceRoom `json:"children"`
InaccessibleChildren []string `json:"inaccessible_children"`
}
// spaceChild is one parsed m.space.child of a room.
type spaceChild struct {
roomID string
via []string
order string
ts int64
suggested bool
}
// localSpaceRoom is a room this server is in, read for a hierarchy.
type localSpaceRoom struct {
room SpaceRoom
children []spaceChild
state auth.State
}
// Hierarchy reads one page of a space's tree.
func (r *API) Hierarchy(ctx context.Context, req HierarchyRequest) (HierarchyPage, error) {
maxDepth := req.MaxDepth
if maxDepth < 0 || maxDepth > MaxHierarchyDepth {
maxDepth = MaxHierarchyDepth
}
limit := req.Limit
if limit <= 0 {
limit = DefaultHierarchyLimit
}
limit = min(limit, MaxHierarchyLimit)
type queued struct {
roomID string
depth int
via []string
}
queue := []queued{{roomID: req.RoomID}}
visited := map[string]bool{}
remote := map[string]*FederationHierarchy{}
var (
out HierarchyPage
emitted int
)
for len(queue) > 0 && len(visited) < maxHierarchyRooms {
next := queue[0]
queue = queue[1:]
if visited[next.roomID] {
continue
}
visited[next.roomID] = true
isRoot := next.roomID == req.RoomID
room, children, accessible, err := r.hierarchyRoom(ctx, req.User, next.roomID, next.via, req.SuggestedOnly, remote)
switch {
case err != nil && isRoot:
return HierarchyPage{}, err
case err != nil:
// A child this server cannot describe — not here, and no
// server it names would say — is left out, as the spec has
// it, rather than failing the walk.
r.log.Debug("left a room out of a space hierarchy",
"space", req.RoomID, "room_id", next.roomID, "error", err)
continue
case !accessible && isRoot:
return HierarchyPage{}, fmt.Errorf("%w: %s may not see the space %s", ErrForbidden, req.User, req.RoomID)
case !accessible:
continue
}
if emitted >= req.From {
if len(out.Rooms) == limit {
out.Next = emitted
return out, nil
}
out.Rooms = append(out.Rooms, room)
}
emitted++
// Only a space's children are walked into: an ordinary room with
// an m.space.child event is not a space, and the event in it is
// an orphan.
if next.depth >= maxDepth || (!isRoot && room.RoomType != RoomTypeSpace) {
continue
}
for _, child := range children {
if !visited[child.roomID] {
queue = append(queue, queued{roomID: child.roomID, depth: next.depth + 1, via: child.via})
}
}
}
return out, nil
}
// hierarchyRoom describes one room of a walk, and reports whether the user
// may know about it.
func (r *API) hierarchyRoom(ctx context.Context, userID, roomID string, via []string, suggestedOnly bool, remote map[string]*FederationHierarchy) (SpaceRoom, []spaceChild, bool, error) {
local, err := r.participating(ctx, roomID)
if err != nil {
return SpaceRoom{}, nil, false, err
}
if local {
read, err := r.readSpaceRoom(ctx, roomID, suggestedOnly)
if err != nil {
return SpaceRoom{}, nil, false, err
}
accessible, err := r.userMayPeek(ctx, read, userID)
if err != nil {
return SpaceRoom{}, nil, false, err
}
read.room.AllowedRoomIDs = nil
return read.room, read.children, accessible, nil
}
answer, err := r.remoteHierarchy(ctx, roomID, via, suggestedOnly, remote)
if err != nil {
return SpaceRoom{}, nil, false, err
}
room := answer.Room
children := parseRemoteChildren(room.ChildrenState, suggestedOnly)
if room.ChildrenState == nil {
room.ChildrenState = []SpaceChildEvent{}
}
// The other server judged the room for this server as a whole; a
// restricted room is shown to the user only if they are in one of the
// rooms it lets in, which only this server knows.
accessible := true
if isRestricted(room.JoinRule) && !isPeekable(room.JoinRule, room.WorldReadable) {
accessible = false
for _, allowed := range room.AllowedRoomIDs {
membership, err := r.store.Membership(ctx, allowed, userID)
if err != nil {
return SpaceRoom{}, nil, false, err
}
if membership.Membership == event.MembershipJoin {
accessible = true
break
}
}
}
room.AllowedRoomIDs = nil
return room, children, accessible, nil
}
// remoteHierarchy asks the servers a child event names about a room,
// remembering the answer for the rest of the walk.
func (r *API) remoteHierarchy(ctx context.Context, roomID string, via []string, suggestedOnly bool, cache map[string]*FederationHierarchy) (*FederationHierarchy, error) {
if cached, ok := cache[roomID]; ok {
if cached == nil {
return nil, fmt.Errorf("%w: %s", ErrNoSuchRoom, roomID)
}
return cached, nil
}
cache[roomID] = nil
if !r.federating() || r.remote == nil {
return nil, fmt.Errorf("%w: %s is not a room this server is in", ErrNoSuchRoom, roomID)
}
var lastErr error
for _, server := range via {
if server == r.serverName {
continue
}
raw, err := r.remote.Hierarchy(ctx, server, roomID, suggestedOnly)
if err != nil {
lastErr = remoteError(server, err)
continue
}
var answer FederationHierarchy
if err := json.Unmarshal(raw, &answer); err != nil || answer.Room.RoomID != roomID {
lastErr = fmt.Errorf("%w: %s sent an unreadable hierarchy for %s", ErrRemote, server, roomID)
continue
}
cache[roomID] = &answer
return &answer, nil
}
if lastErr == nil {
lastErr = fmt.Errorf("%w: no server to ask about %s", ErrNoSuchRoom, roomID)
}
return nil, lastErr
}
// FederatedHierarchy answers another server walking a space of this one.
func (r *API) FederatedHierarchy(ctx context.Context, origin, roomID string, suggestedOnly bool) (FederationHierarchy, error) {
local, err := r.participating(ctx, roomID)
if err != nil {
return FederationHierarchy{}, err
}
if !local {
return FederationHierarchy{}, fmt.Errorf("%w: %s", ErrNoSuchRoom, roomID)
}
root, err := r.readSpaceRoom(ctx, roomID, suggestedOnly)
if err != nil {
return FederationHierarchy{}, err
}
accessible, err := r.serverMayPeek(ctx, root, origin)
if err != nil {
return FederationHierarchy{}, err
}
if !accessible {
// Not found rather than forbidden: whether this server is in a
// room the asker may not see is itself not theirs to know.
return FederationHierarchy{}, fmt.Errorf("%w: %s", ErrNoSuchRoom, roomID)
}
out := FederationHierarchy{Room: root.room, Children: []SpaceRoom{}, InaccessibleChildren: []string{}}
for _, child := range root.children {
local, err := r.participating(ctx, child.roomID)
if err != nil {
return FederationHierarchy{}, err
}
if !local {
continue
}
read, err := r.readSpaceRoom(ctx, child.roomID, suggestedOnly)
if err != nil {
return FederationHierarchy{}, err
}
ok, err := r.serverMayPeek(ctx, read, origin)
if err != nil {
return FederationHierarchy{}, err
}
if !ok {
out.InaccessibleChildren = append(out.InaccessibleChildren, child.roomID)
continue
}
read.room.ChildrenState = nil
out.Children = append(out.Children, read.room)
}
return out, nil
}
// RoomSummaryRequest asks for the public face of one room.
type RoomSummaryRequest struct {
RoomID string
// User is who is asking, empty for a guest without a token.
User string
// Via are servers to ask when this one is not in the room.
Via []string
}
// RoomSummaryResponse is GET /room_summary's answer: a hierarchy room
// without its children, plus where the user stands.
type RoomSummaryResponse struct {
SpaceRoom
Membership string `json:"membership,omitempty"`
}
// RoomSummary describes one room to a user who may not be in it, as a link
// preview or an invite needs it.
func (r *API) RoomSummary(ctx context.Context, req RoomSummaryRequest) (RoomSummaryResponse, error) {
room, _, accessible, err := r.hierarchyRoom(ctx, req.User, req.RoomID, req.Via, false, map[string]*FederationHierarchy{})
if err != nil {
return RoomSummaryResponse{}, err
}
if !accessible {
return RoomSummaryResponse{}, fmt.Errorf("%w: %s may not see %s", ErrForbidden, req.User, req.RoomID)
}
room.ChildrenState = nil
out := RoomSummaryResponse{SpaceRoom: room}
if req.User != "" {
membership, err := r.store.Membership(ctx, req.RoomID, req.User)
if err != nil && !isNotFound(err) {
return RoomSummaryResponse{}, err
}
out.Membership = membership.Membership
if out.Membership == "" {
out.Membership = event.MembershipLeave
}
}
return out, nil
}
// readSpaceRoom reads a room this server is in.
func (r *API) readSpaceRoom(ctx context.Context, roomID string, suggestedOnly bool) (localSpaceRoom, error) {
room, rules, err := r.roomAndRules(ctx, roomID)
if err != nil {
return localSpaceRoom{}, err
}
summary, err := r.store.RoomSummary(ctx, roomID)
if err != nil {
return localSpaceRoom{}, err
}
st, err := r.currentState(ctx, room, rules)
if err != nil {
return localSpaceRoom{}, err
}
out := localSpaceRoom{
state: st,
room: SpaceRoom{
RoomID: roomID,
Name: summary.Name,
Topic: summary.Topic,
AvatarURL: summary.AvatarURL,
CanonicalAlias: summary.CanonicalAlias,
NumJoinedMembers: summary.JoinedMembers,
WorldReadable: summary.HistoryVisibility == HistoryVisibilityWorldReadable,
GuestCanJoin: summary.GuestAccess == "can_join",
JoinRule: cmp.Or(st.JoinRule(), auth.JoinRuleInvite),
RoomType: summary.RoomType,
RoomVersion: room.Version,
ChildrenState: []SpaceChildEvent{},
},
}
if enc := st.Get(TypeEncryption, ""); enc != nil {
var content struct {
Algorithm string `json:"algorithm"`
}
if json.Unmarshal(enc.Content(), &content) == nil {
out.room.Encryption = content.Algorithm
}
}
if isRestricted(out.room.JoinRule) {
if out.room.AllowedRoomIDs, err = allowedRooms(st); err != nil {
return localSpaceRoom{}, err
}
}
for key, e := range st {
if key.Type != TypeSpaceChild {
continue
}
child, ok := parseSpaceChild(key.StateKey, e.Content(), e.OriginServerTS())
if !ok || (suggestedOnly && !child.suggested) {
continue
}
out.children = append(out.children, child)
out.room.ChildrenState = append(out.room.ChildrenState, SpaceChildEvent{
Type: TypeSpaceChild,
StateKey: key.StateKey,
Sender: e.Sender(),
Content: e.Content(),
OriginServerTS: e.OriginServerTS(),
})
}
sortChildren(out.children)
slices.SortFunc(out.room.ChildrenState, func(a, b SpaceChildEvent) int {
return strings.Compare(a.StateKey, b.StateKey)
})
return out, nil
}
// userMayPeek reports whether a user may know about a room they are not
// necessarily in: the spec's accessibility rule for a hierarchy.
func (r *API) userMayPeek(ctx context.Context, read localSpaceRoom, userID string) (bool, error) {
switch read.state.Membership(userID) {
case event.MembershipJoin, event.MembershipInvite:
return true, nil
}
if isPeekable(read.room.JoinRule, read.room.WorldReadable) {
return true, nil
}
if isRestricted(read.room.JoinRule) {
for _, allowed := range read.room.AllowedRoomIDs {
membership, err := r.store.Membership(ctx, allowed, userID)
if err != nil {
return false, err
}
if membership.Membership == event.MembershipJoin {
return true, nil
}
}
}
return false, nil
}
// serverMayPeek is userMayPeek for another server: anything a user of it
// could be shown. A restricted room is shown with its allow list, which
// the asking server checks against its own users.
func (r *API) serverMayPeek(ctx context.Context, read localSpaceRoom, origin string) (bool, error) {
if isPeekable(read.room.JoinRule, read.room.WorldReadable) || isRestricted(read.room.JoinRule) {
return true, nil
}
for key, e := range read.state {
if key.Type != event.TypeMember {
continue
}
server, err := id.ServerNameOf(key.StateKey)
if err != nil || server != origin {
continue
}
if membership, err := e.Membership(); err == nil &&
(membership == event.MembershipJoin || membership == event.MembershipInvite) {
return true, nil
}
}
return false, nil
}
// isPeekable reports whether anybody at all may know about a room.
func isPeekable(joinRule string, worldReadable bool) bool {
switch joinRule {
case auth.JoinRulePublic, auth.JoinRuleKnock, auth.JoinRuleKnockRestricted:
return true
}
return worldReadable
}
func isRestricted(joinRule string) bool {
return joinRule == auth.JoinRuleRestricted || joinRule == auth.JoinRuleKnockRestricted
}
// parseSpaceChild reads one m.space.child. A child event without a
// non-empty `via` is how a space removes a child — the event cannot be
// deleted, only emptied — so it does not count.
func parseSpaceChild(stateKey string, content json.RawMessage, ts int64) (spaceChild, bool) {
if !strings.HasPrefix(stateKey, "!") {
return spaceChild{}, false
}
var c struct {
Via []any `json:"via"`
Order any `json:"order"`
Suggested any `json:"suggested"`
}
if err := json.Unmarshal(content, &c); err != nil {
return spaceChild{}, false
}
child := spaceChild{roomID: stateKey, ts: ts}
for _, v := range c.Via {
if s, ok := v.(string); ok && s != "" {
child.via = append(child.via, s)
}
}
if len(child.via) == 0 {
return spaceChild{}, false
}
if order, ok := c.Order.(string); ok && validOrder(order) {
child.order = order
}
child.suggested, _ = c.Suggested.(bool)
return child, true
}
// parseRemoteChildren reads the children of a room another server
// described.
func parseRemoteChildren(state []SpaceChildEvent, suggestedOnly bool) []spaceChild {
var out []spaceChild
for _, e := range state {
if e.Type != TypeSpaceChild {
continue
}
child, ok := parseSpaceChild(e.StateKey, e.Content, e.OriginServerTS)
if !ok || (suggestedOnly && !child.suggested) {
continue
}
out = append(out, child)
}
sortChildren(out)
return out
}
// validOrder reports whether an order string may sort a child: printable
// ASCII and no longer than the spec allows.
func validOrder(order string) bool {
if order == "" || len(order) > maxOrderLength {
return false
}
for i := range len(order) {
if order[i] < 0x20 || order[i] > 0x7e {
return false
}
}
return true
}
// sortChildren puts children in the spec's order: those with an order
// first, by it; then by when the child event was sent; then by room ID,
// which only breaks ties.
func sortChildren(children []spaceChild) {
slices.SortStableFunc(children, func(a, b spaceChild) int {
switch {
case a.order != "" && b.order == "":
return -1
case a.order == "" && b.order != "":
return 1
}
return cmp.Or(
strings.Compare(a.order, b.order),
cmp.Compare(a.ts, b.ts),
strings.Compare(a.roomID, b.roomID),
)
})
}
+10 -13
View File
@@ -6,11 +6,14 @@ import (
"errors"
"fmt"
"go.opentelemetry.io/otel/attribute"
"github.com/dariuszjaniak/warp/server/internal/matrix/auth"
"github.com/dariuszjaniak/warp/server/internal/matrix/event"
"github.com/dariuszjaniak/warp/server/internal/matrix/roomversion"
"github.com/dariuszjaniak/warp/server/internal/matrix/stateres"
"github.com/dariuszjaniak/warp/server/internal/storage"
"github.com/dariuszjaniak/warp/server/internal/tracing"
)
// sendRequest is one event this server is about to create.
@@ -344,20 +347,17 @@ type persistRequest struct {
}
// persist writes the event and everything its arrival changes, then emits.
func (r *API) persist(ctx context.Context, req persistRequest) error {
func (r *API) persist(ctx context.Context, req persistRequest) (err error) {
ctx, span := tracing.Start(ctx, "roomserver.persist",
attribute.String("matrix.room_id", req.Room.ID),
attribute.String("matrix.event_type", req.Event.Type()))
defer func() { tracing.End(span, err) }()
e := req.Event
stateKey, isState := e.StateKey()
write := storage.RoomWrite{
Event: storage.StoredEvent{
RoomID: req.Room.ID,
EventID: e.EventID(),
Type: e.Type(),
Sender: e.Sender(),
Depth: e.Depth(),
OriginServerTS: e.OriginServerTS(),
JSON: e.JSON(),
},
Event: storedOf(req.Room.ID, e),
StateBefore: stateEntries(req.StateBefore),
Extremities: req.Extremities,
Transaction: req.Transaction,
@@ -365,9 +365,6 @@ func (r *API) persist(ctx context.Context, req persistRequest) error {
if write.Extremities == nil {
write.Extremities = []string{e.EventID()}
}
if isState {
write.Event.StateKey = &stateKey
}
// The state after this event. For the ordinary case that is one tuple
// to upsert; after a resolution it is the whole state, which the store
+5 -1
View File
@@ -129,7 +129,11 @@ func (r *API) Event(ctx context.Context, roomID, userID, eventID string) (Client
if err != nil {
return ClientEvent{}, err
}
return r.clientEvent(stored, e, roomID), nil
out := []ClientEvent{r.clientEvent(stored, e, roomID)}
if err := r.bundleAggregations(ctx, roomID, rules, userID, visible, []storage.StoredEvent{stored}, out); err != nil {
return ClientEvent{}, err
}
return out[0], nil
}
// RoomState returns a room's state in the client shape: the current state
+475
View File
@@ -0,0 +1,475 @@
package roomserver
import (
"context"
"encoding/json"
"fmt"
"slices"
"github.com/dariuszjaniak/warp/server/internal/matrix/event"
"github.com/dariuszjaniak/warp/server/internal/matrix/roomversion"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// Relations: threads, edits, reactions and references.
//
// An event relates to another through content.m.relates_to, and nothing
// about that is checked by the authorisation rules — a relation is a
// client convention the server indexes, not part of the room's DAG. The
// server's part is three things: recording which event points at which
// (relationOf, written with the event), listing an event's children
// (/relations and /threads), and summarising them into the parent's
// unsigned.m.relations wherever the parent is served (the bundled
// aggregations), so that a client rendering a timeline knows a message
// has a thread under it or has been edited without asking about every
// event.
// relationOf reads content.m.relates_to. Both fields have to be strings,
// and an m.in_reply_to alone is not a relation in this sense: a reply
// points at an event without being one of its children.
func relationOf(content json.RawMessage) (eventID, relType string) {
var probe struct {
RelatesTo *struct {
EventID any `json:"event_id"`
RelType any `json:"rel_type"`
} `json:"m.relates_to"`
}
if err := json.Unmarshal(content, &probe); err != nil || probe.RelatesTo == nil {
return "", ""
}
id, ok1 := probe.RelatesTo.EventID.(string)
rel, ok2 := probe.RelatesTo.RelType.(string)
if !ok1 || !ok2 || id == "" || rel == "" {
return "", ""
}
return id, rel
}
// ThreadOf is the thread an event is in: the root its m.relates_to names
// when the relation is m.thread, or "" for an event on the main timeline.
// The push server files each notification under it.
func ThreadOf(e *event.Event) string {
if root, rel := relationOf(e.Content()); rel == storage.RelThread {
return root
}
return ""
}
// searchableText is what the search index holds for an event: the body of
// a message, the name of a room or its topic, as the spec's three search
// keys. An encrypted message has no body the server can read and is never
// found, which is what end-to-end encryption means.
func searchableText(e *event.Event) (key, text string) {
var field, k string
switch e.Type() {
case TypeMessage:
field, k = "body", "content.body"
case TypeName:
field, k = "name", "content.name"
case TypeTopic:
field, k = "topic", "content.topic"
default:
return "", ""
}
var content map[string]any
if err := json.Unmarshal(e.Content(), &content); err != nil {
return "", ""
}
value, ok := content[field].(string)
if !ok || value == "" {
return "", ""
}
return k, value
}
// maxRecursionDepth is how far a recursive /relations walks: the spec's
// own suggestion, which covers a reaction to an edit in a thread.
const maxRecursionDepth = 3
// maxReferenceChunk caps the event IDs in an m.reference summary.
const maxReferenceChunk = 50
// RelationsRequest is one page of an event's children.
type RelationsRequest struct {
RoomID string
EventID string
// User is who is asking. The parent has to be visible to them, and
// so does every child.
User string
// RelType and EventType narrow the children, when set.
RelType string
EventType string
// Low, High and Direction are a TimelineRequest's.
Low int64
High int64
Direction storage.TimelineDirection
Limit int
// Recurse includes the children's children, down to
// maxRecursionDepth levels.
Recurse bool
}
// RelationsPage is one page of an event's children.
type RelationsPage struct {
Timeline
// Depth is how many levels the read went down: 1 without recursion.
Depth int
}
// Relations reads one page of an event's children.
func (r *API) Relations(ctx context.Context, req RelationsRequest) (RelationsPage, error) {
_, rules, err := r.roomAndRules(ctx, req.RoomID)
if err != nil {
return RelationsPage{}, err
}
visible, err := r.requireReadable(ctx, req.RoomID, req.User)
if err != nil {
return RelationsPage{}, err
}
parent, err := r.store.Event(ctx, req.RoomID, req.EventID)
if err != nil {
return RelationsPage{}, err
}
if parent.Outlier || !visible.Contains(parent.StreamOrdering) {
return RelationsPage{}, fmt.Errorf("event %q: %w", req.EventID, ErrNoSuchEvent)
}
// The parents to read the children of: the event, and with recursion
// every descendant down to the depth limit. The filters apply at every
// level, as the spec has it, so a walk for m.thread children only
// follows threads.
//
// The children of one level are the next level, so the walk adds a
// level to the parents only while it is above the limit: the events at
// the limit are read as children and are nobody's parents. depth is
// the deepest level that had anything in it.
parents := []string{req.EventID}
depth := 1
if req.Recurse {
depth = 0
level := parents
seen := map[string]bool{req.EventID: true}
for depth < maxRecursionDepth {
children, err := r.store.ChildEventIDs(ctx, req.RoomID, level, req.RelType, req.EventType)
if err != nil {
return RelationsPage{}, err
}
// A relation loop cannot be built — an event ID is a hash of
// content naming its parent — but a repeat across levels can.
children = slices.DeleteFunc(children, func(id string) bool {
if seen[id] {
return true
}
seen[id] = true
return false
})
if len(children) == 0 {
break
}
depth++
if depth < maxRecursionDepth {
parents = append(parents, children...)
}
level = children
}
depth = max(depth, 1)
}
empty := Timeline{Start: req.Low, End: req.Low}
if req.Direction == storage.DirectionBackward && req.High != 0 {
empty = Timeline{Start: req.High, End: req.High}
}
low, high, any := visible.Clamp(req.Low, req.High)
if !any || req.Limit <= 0 {
return RelationsPage{Timeline: empty, Depth: depth}, nil
}
stored, err := r.store.Relations(ctx, storage.RelationsRequest{
RoomID: req.RoomID,
ParentIDs: parents,
RelType: req.RelType,
EventType: req.EventType,
Low: low,
High: high,
Direction: req.Direction,
Limit: req.Limit + 1,
})
if err != nil {
return RelationsPage{}, err
}
out := RelationsPage{Timeline: empty, Depth: depth}
if len(stored) > req.Limit {
out.Limited = true
stored = stored[:req.Limit]
}
if len(stored) == 0 {
return out, nil
}
if req.Direction == storage.DirectionBackward {
slices.Reverse(stored)
}
if out.Events, err = r.clientEvents(stored, rules, req.RoomID); err != nil {
return RelationsPage{}, err
}
if err := r.bundleAggregations(ctx, req.RoomID, rules, req.User, visible, stored, out.Events); err != nil {
return RelationsPage{}, err
}
out.Start = storage.PositionBefore(stored[0].StreamOrdering)
out.End = stored[len(stored)-1].StreamOrdering
return out, nil
}
// ThreadsRequest is one page of a room's threads.
type ThreadsRequest struct {
RoomID string
User string
// Participated keeps only the threads the user started or replied in.
Participated bool
// Before is where the previous page ended, zero for the first.
Before int64
Limit int
}
// ThreadsPage is one page of a room's threads: their root events, newest
// activity first, each with its m.thread summary.
type ThreadsPage struct {
Roots []ClientEvent
// Next is the Before of the next page, zero at the end.
Next int64
}
// Threads reads one page of a room's threads.
func (r *API) Threads(ctx context.Context, req ThreadsRequest) (ThreadsPage, error) {
_, rules, err := r.roomAndRules(ctx, req.RoomID)
if err != nil {
return ThreadsPage{}, err
}
visible, err := r.requireReadable(ctx, req.RoomID, req.User)
if err != nil {
return ThreadsPage{}, err
}
if req.Limit <= 0 {
return ThreadsPage{}, nil
}
participant := ""
if req.Participated {
participant = req.User
}
threads, err := r.store.Threads(ctx, storage.ThreadsRequest{
RoomID: req.RoomID,
ParticipatedBy: participant,
Before: req.Before,
High: visible.To,
Limit: req.Limit + 1,
})
if err != nil {
return ThreadsPage{}, err
}
var out ThreadsPage
if len(threads) > req.Limit {
threads = threads[:req.Limit]
out.Next = threads[len(threads)-1].LatestOrdering
}
ids := make([]string, 0, len(threads))
for _, t := range threads {
ids = append(ids, t.RootID)
}
stored, err := r.store.EventsByID(ctx, req.RoomID, ids)
if err != nil {
return ThreadsPage{}, err
}
byID := make(map[string]storage.StoredEvent, len(stored))
for _, s := range stored {
byID[s.EventID] = s
}
// In the order the store ranked them, leaving out any root the user may
// not see: a thread started before a `joined` user's join is not
// theirs to read, even if it is still going.
roots := make([]storage.StoredEvent, 0, len(threads))
for _, t := range threads {
s, ok := byID[t.RootID]
if !ok || s.Outlier || !visible.Contains(s.StreamOrdering) {
continue
}
roots = append(roots, s)
}
if out.Roots, err = r.clientEvents(roots, rules, req.RoomID); err != nil {
return ThreadsPage{}, err
}
if err := r.bundleAggregations(ctx, req.RoomID, rules, req.User, visible, roots, out.Roots); err != nil {
return ThreadsPage{}, err
}
return out, nil
}
// threadSummary is unsigned.m.relations.m.thread.
type threadSummary struct {
LatestEvent ClientEvent `json:"latest_event"`
Count int `json:"count"`
CurrentUserParticipated bool `json:"current_user_participated"`
}
// referenceSummary is unsigned.m.relations.m.reference.
type referenceSummary struct {
Chunk []referenceEntry `json:"chunk"`
}
type referenceEntry struct {
EventID string `json:"event_id"`
}
// bundleAggregations adds unsigned.m.relations to the events of a read:
// the thread under each of them, its latest edit, and what references it.
//
// stored and events are the same events, in the same order — the rows and
// their client shape — because the summaries need what only the rows have
// (whether an event was redacted) and are written into the client shape.
// Annotations are not summarised: the spec dropped their aggregation, and a
// client counts reactions from the events themselves.
func (r *API) bundleAggregations(ctx context.Context, roomID string, rules roomversion.Rules, userID string, visible VisibleRange, stored []storage.StoredEvent, events []ClientEvent) error {
if len(stored) == 0 {
return nil
}
ids := make([]string, 0, len(stored))
parents := make(map[string]storage.StoredEvent, len(stored))
for _, s := range stored {
ids = append(ids, s.EventID)
parents[s.EventID] = s
}
aggregates, err := r.store.RelationAggregates(ctx, roomID, ids, visible.To)
if err != nil {
return err
}
if len(aggregates) == 0 {
return nil
}
type thread struct {
count int
latest int64
latestID string
participated bool
}
threads := map[string]*thread{}
edits := map[string]storage.RelationAggregate{}
referenced := map[string]bool{}
for _, a := range aggregates {
parent := parents[a.ParentID]
switch a.RelType {
case storage.RelThread:
t := threads[a.ParentID]
if t == nil {
t = &thread{participated: parent.Sender == userID}
threads[a.ParentID] = t
}
t.count += a.Count
if a.LatestOrdering > t.latest {
t.latest, t.latestID = a.LatestOrdering, a.LatestEventID
}
if a.Sender == userID {
t.participated = true
}
case storage.RelReplace:
// Only the original's sender may edit it, a state event cannot
// be edited, and a redacted event's edits are moot: what they
// replaced is gone.
if a.Sender != parent.Sender || parent.IsState() || parent.RedactedBy != "" {
continue
}
edits[a.ParentID] = a
case storage.RelReference:
referenced[a.ParentID] = true
}
}
// The latest thread replies and edits, read in one go.
latestIDs := make([]string, 0, len(threads)+len(edits))
for _, t := range threads {
latestIDs = append(latestIDs, t.latestID)
}
for _, a := range edits {
latestIDs = append(latestIDs, a.LatestEventID)
}
latest, err := r.clientEventsByID(ctx, roomID, rules, latestIDs)
if err != nil {
return err
}
latestByID := make(map[string]ClientEvent, len(latest))
for _, e := range latest {
latestByID[e.EventID] = e
}
var references map[string][]string
if len(referenced) > 0 {
refParents := make([]string, 0, len(referenced))
for id := range referenced {
refParents = append(refParents, id)
}
if references, err = r.store.RelationEventIDs(ctx, roomID, refParents,
storage.RelReference, visible.To, maxReferenceChunk); err != nil {
return err
}
}
for i := range events {
id := events[i].EventID
relations := map[string]any{}
if t, ok := threads[id]; ok {
if e, ok := latestByID[t.latestID]; ok {
relations[storage.RelThread] = threadSummary{
LatestEvent: e,
Count: t.count,
CurrentUserParticipated: t.participated,
}
}
}
if a, ok := edits[id]; ok {
// An edit has to keep the type of what it edits; one that does
// not is not a replacement the client should render.
if e, ok := latestByID[a.LatestEventID]; ok && e.Type == events[i].Type {
relations[storage.RelReplace] = e
}
}
if refs := references[id]; len(refs) > 0 {
summary := referenceSummary{Chunk: make([]referenceEntry, 0, len(refs))}
for _, ref := range refs {
summary.Chunk = append(summary.Chunk, referenceEntry{EventID: ref})
}
relations[storage.RelReference] = summary
}
if len(relations) == 0 {
continue
}
events[i].Unsigned = r.withUnsigned(events[i].Unsigned, "m.relations", relations)
}
return nil
}
// withUnsigned sets one key of an event's unsigned block.
func (r *API) withUnsigned(unsigned json.RawMessage, key string, value any) json.RawMessage {
block := map[string]json.RawMessage{}
if len(unsigned) > 0 {
if err := json.Unmarshal(unsigned, &block); err != nil {
r.log.Warn("an unsigned block this server built is unreadable", "error", err)
block = map[string]json.RawMessage{}
}
}
block[key] = mustMarshal(value)
return mustMarshal(block)
}
+3
View File
@@ -37,6 +37,7 @@ type RemoteServers interface {
Backfill(ctx context.Context, destination, roomID string, from []string, limit int) (federation.Transaction, error)
QueryDirectory(ctx context.Context, destination, alias string) (federation.DirectoryResponse, error)
PublicRooms(ctx context.Context, destination string, limit int, since, search string) (json.RawMessage, error)
Hierarchy(ctx context.Context, destination, roomID string, suggestedOnly bool) (json.RawMessage, error)
}
// federating reports whether this server may talk to others right now.
@@ -226,6 +227,8 @@ func storedOf(roomID string, e *event.Event) storage.StoredEvent {
if key, ok := e.StateKey(); ok {
s.StateKey = &key
}
s.RelatesTo, s.RelType = relationOf(e.Content())
s.SearchKey, s.SearchText = searchableText(e)
return s
}
+122
View File
@@ -0,0 +1,122 @@
package roomserver
import (
"context"
"slices"
"github.com/dariuszjaniak/warp/server/internal/matrix/event"
"github.com/dariuszjaniak/warp/server/internal/storage"
"github.com/dariuszjaniak/warp/server/internal/tracing"
)
// Full-text search: the part of /search that decides where a user may
// look.
//
// The index holds every searchable event of every room on the server, and
// the query against it is the store's. What is decided here is the set of
// rooms, each bounded to what the user may read of it — the same visible
// range a timeline read is clamped to, so a user who left a room finds what
// was said while they were in it and nothing since, and a user who joined a
// `joined` room finds nothing from before they came.
// SearchRequest is one page of a search.
type SearchRequest struct {
User string
Term string
// Keys are the fields to match; empty means all three.
Keys []string
// Rooms and NotRooms narrow the rooms searched, as a filter's rooms
// and not_rooms do.
Rooms []string
NotRooms []string
// Filter narrows by type and sender.
Filter storage.TimelineFilter
Order storage.SearchOrder
Offset int
Before int64
Limit int
}
// SearchResult is one event a search found.
type SearchResult struct {
Event ClientEvent
// Position is the event's stream position, which a search by recency
// pages by.
Position int64
Rank float64
}
// Search runs one page of a search.
func (r *API) Search(ctx context.Context, req SearchRequest) (_ []SearchResult, _ int, err error) {
ctx, span := tracing.Start(ctx, "roomserver.search")
defer func() { tracing.End(span, err) }()
memberships, err := r.UserMemberships(ctx, req.User,
[]string{event.MembershipJoin, event.MembershipLeave, event.MembershipBan})
if err != nil {
return nil, 0, err
}
ranges := make([]storage.SearchRange, 0, len(memberships))
for _, m := range memberships {
if len(req.Rooms) > 0 && !slices.Contains(req.Rooms, m.RoomID) {
continue
}
if slices.Contains(req.NotRooms, m.RoomID) {
continue
}
visible, err := r.VisibleRange(ctx, m.RoomID, req.User)
if err != nil {
return nil, 0, err
}
if !visible.Allowed {
continue
}
ranges = append(ranges, storage.SearchRange{RoomID: m.RoomID, Low: visible.From, High: visible.To})
}
if len(ranges) == 0 {
return nil, 0, nil
}
found, total, err := r.store.Search(ctx, storage.SearchRequest{
Term: req.Term,
Rooms: ranges,
Keys: req.Keys,
Filter: req.Filter,
Order: req.Order,
Offset: req.Offset,
Before: req.Before,
Limit: req.Limit,
})
if err != nil {
return nil, 0, err
}
out := make([]SearchResult, 0, len(found))
for _, f := range found {
room, err := r.store.Room(ctx, f.Event.RoomID)
if err != nil {
return nil, 0, err
}
rules, err := r.rules(room)
if err != nil {
return nil, 0, err
}
e, err := parseEvent(f.Event, rules)
if err != nil {
return nil, 0, err
}
out = append(out, SearchResult{
Event: r.clientEvent(f.Event, e, f.Event.RoomID),
Position: f.Event.StreamOrdering,
Rank: f.Rank,
})
}
return out, total, nil
}
+159
View File
@@ -0,0 +1,159 @@
package roomserver
import (
"context"
"slices"
"strings"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// The reads sliding sync needs that /sync does not: a room list ordered by
// activity, each room's public face, and pieces of a room's state picked
// tuple by tuple rather than all of it.
//
// A classic sync sends a room's whole state once and then every change;
// sliding sync sends only the tuples the client asked for, which for a room
// list is a name and an avatar out of a state that may hold ten thousand
// member events. Reading the tuples before the event bodies is what makes
// that cheap.
// bumpEventTypes are the events that move a room up a room list: the
// things a user would call activity. A membership change, a reaction or a
// receipt is not, or a busy room would sit on top of the list for reasons
// nobody looking at it could see.
var bumpEventTypes = []string{
"m.room.create",
TypeMessage,
"m.room.encrypted",
"m.sticker",
"m.call.invite",
"m.poll.start",
"m.beacon_info",
}
// BumpStamps returns each room's latest activity at or before to.
func (r *API) BumpStamps(ctx context.Context, roomIDs []string, to int64) (map[string]int64, error) {
return r.store.LatestEventPositions(ctx, roomIDs, bumpEventTypes, to)
}
// RoomListEntry is what a room list shows of a room.
type RoomListEntry struct {
storage.RoomSummary
// Encrypted reports an m.room.encryption in the room's state.
Encrypted bool
}
// RoomListEntry reads one room's public face.
func (r *API) RoomListEntry(ctx context.Context, roomID string) (RoomListEntry, error) {
summary, err := r.store.RoomSummary(ctx, roomID)
if err != nil {
return RoomListEntry{}, err
}
out := RoomListEntry{RoomSummary: summary}
if _, err := r.store.CurrentStateEvent(ctx, roomID, TypeEncryption, ""); err == nil {
out.Encrypted = true
} else if !isNotFound(err) {
return RoomListEntry{}, err
}
return out, nil
}
// RequiredStateRequest picks pieces of a room's state.
type RequiredStateRequest struct {
RoomID string
User string
// Since, when set, asks only for the tuples that changed after it: the
// incremental half of a sliding sync. Zero is the state in full.
Since int64
// To is the position the state is read at, clamped to what the user
// may see. Zero is the top of the stream.
To int64
// Match picks the tuples.
Match func(eventType, stateKey string) bool
}
// RequiredState reads the tuples of a room's state that Match picks.
func (r *API) RequiredState(ctx context.Context, req RequiredStateRequest) ([]ClientEvent, error) {
room, rules, err := r.roomAndRules(ctx, req.RoomID)
if err != nil {
return nil, err
}
visible, err := r.requireReadable(ctx, room.ID, req.User)
if err != nil {
return nil, err
}
to := req.To
if visible.To != 0 && (to == 0 || to > visible.To) {
to = visible.To
}
var after []storage.StateEntry
if to == 0 {
after, err = r.store.CurrentState(ctx, req.RoomID)
} else {
after, err = r.store.StateAtStream(ctx, req.RoomID, to)
}
if err != nil {
return nil, err
}
var before map[[2]string]string
if req.Since != 0 {
entries, err := r.store.StateAtStream(ctx, req.RoomID, req.Since)
if err != nil {
return nil, err
}
before = make(map[[2]string]string, len(entries))
for _, e := range entries {
before[[2]string{e.Type, e.StateKey}] = e.EventID
}
}
var ids []string
for _, e := range after {
if !req.Match(e.Type, e.StateKey) {
continue
}
if before != nil && before[[2]string{e.Type, e.StateKey}] == e.EventID {
continue
}
ids = append(ids, e.EventID)
}
if len(ids) == 0 {
return []ClientEvent{}, nil
}
events, err := r.clientEventsByID(ctx, req.RoomID, rules, ids)
if err != nil {
return nil, err
}
slices.SortFunc(events, func(a, b ClientEvent) int {
if c := strings.Compare(a.Type, b.Type); c != 0 {
return c
}
return strings.Compare(*a.StateKey, *b.StateKey)
})
return events, nil
}
// SpaceChildRooms lists the rooms a space this server is in names as its
// children.
func (r *API) SpaceChildRooms(ctx context.Context, spaceID string) ([]string, error) {
local, err := r.participating(ctx, spaceID)
if err != nil || !local {
return nil, err
}
read, err := r.readSpaceRoom(ctx, spaceID, false)
if err != nil {
return nil, err
}
out := make([]string, 0, len(read.children))
for _, c := range read.children {
out = append(out, c.roomID)
}
return out, nil
}
+3
View File
@@ -215,6 +215,9 @@ func (r *API) timeline(ctx context.Context, rules roomversion.Rules, visible Vis
}
out.Events = append(out.Events, r.clientEvent(s, e, req.RoomID))
}
if err := r.bundleAggregations(ctx, req.RoomID, rules, req.User, visible, stored, out.Events); err != nil {
return Timeline{}, err
}
// Start sits below the oldest event, so that handing it back as a High
// bound returns the page before this one; End is at the newest, so that
// handing it back as a Low bound returns the page after. Below the
+90
View File
@@ -0,0 +1,90 @@
package setup
import (
"context"
"database/sql"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/dariuszjaniak/warp/server/internal/metrics"
"github.com/dariuszjaniak/warp/server/internal/roomserver"
"github.com/dariuszjaniak/warp/server/internal/syncapi"
)
// The homeserver's own metrics, beside the HTTP and runtime ones every
// route gets: what a dashboard (deploy/grafana) shows about a server's
// rooms rather than its requests.
//
// They are gauges read on scrape and one counter fed from the roomserver's
// emit stage, so that none of them costs anything between scrapes.
// eventCounter counts persisted events from the emit stage.
type eventCounter struct {
events *prometheus.CounterVec
}
// EventPersisted implements roomserver.Notifier.
func (c *eventCounter) EventPersisted(_ context.Context, n roomserver.Notification) {
origin := "local"
if n.Origin != "" {
origin = "federation"
}
kind := "message"
if n.Event != nil && n.Event.IsState() {
kind = "state"
}
c.events.WithLabelValues(origin, kind).Inc()
}
// registerServerMetrics adds the homeserver's collectors to the registry,
// and returns the event counter to put on the emit stage.
func (s *Server) registerServerMetrics(notifier *syncapi.Notifier) *eventCounter {
counter := &eventCounter{events: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: metrics.Namespace,
Name: "events_persisted_total",
Help: "Room events persisted, by where they came from and whether they are state.",
}, []string{"origin", "kind"})}
s.metrics.MustRegister(counter.events)
s.metrics.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Namespace: metrics.Namespace,
Subsystem: "sync",
Name: "waiting",
Help: "Syncs blocked in a long poll, classic and sliding.",
}, func() float64 { return float64(notifier.Listeners()) }))
s.metrics.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Namespace: metrics.Namespace,
Subsystem: "sync",
Name: "sliding_connections",
Help: "Sliding sync connections whose state the server is holding.",
}, func() float64 {
if s.sync == nil {
return 0
}
return float64(s.sync.SlidingConnections())
}))
s.metrics.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Namespace: metrics.Namespace,
Name: "federation_enabled",
Help: "1 while federation is switched on.",
}, func() float64 {
if s.settings != nil && s.settings.FederationEnabled() {
return 1
}
return 0
}))
return counter
}
// registerDBMetrics adds the connection pool's statistics, for a driver
// that is built on database/sql.
func (s *Server) registerDBMetrics() {
pool, ok := s.db.(interface{ DB() *sql.DB })
if !ok {
return
}
s.metrics.MustRegister(collectors.NewDBStatsCollector(pool.DB(), s.db.Driver()))
}
+170
View File
@@ -0,0 +1,170 @@
package setup_test
import (
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/dariuszjaniak/warp/server/internal/config"
)
// The Phase 10 tests: the PocketBase driver and the new endpoints, through
// the whole stack.
// TestPocketBaseServer runs a server on the pocketbase driver: accounts,
// rooms, search and media, with the media landing in PocketBase's own
// filesystem rather than under media.base_path.
func TestPocketBaseServer(t *testing.T) {
dir := t.TempDir()
dataDir := filepath.Join(dir, "pb_data")
mediaDir := filepath.Join(dir, "media")
srv := newSigningServer(t, func(cfg *config.Config) {
cfg.Database.Driver = "pocketbase"
cfg.Database.ConnectionString = dataDir
cfg.Registration.Enabled = true
cfg.ClientAPI.RateLimit.Enabled = false
cfg.Media.BasePath = mediaDir
})
health := get(t, srv, http.MethodGet, "/healthz")
if health.Code != http.StatusOK || !strings.Contains(health.Body.String(), `"database":"pocketbase"`) {
t.Fatalf("healthz: %d %s", health.Code, health.Body.String())
}
token := registerThrough(t, srv, "alice")
rec := post(t, srv, "/_matrix/client/v3/createRoom", map[string]any{"name": "On PocketBase"}, token)
if rec.Code != http.StatusOK {
t.Fatalf("createRoom: %d %s", rec.Code, rec.Body.String())
}
roomID := decode[map[string]string](t, rec)["room_id"]
rec = send(t, srv, http.MethodPut,
"/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/send/m.room.message/t1",
"application/json", []byte(`{"msgtype":"m.text","body":"stored in a pocket"}`), token, nil)
if rec.Code != http.StatusOK {
t.Fatalf("send: %d %s", rec.Code, rec.Body.String())
}
rec = post(t, srv, "/_matrix/client/v3/search", map[string]any{
"search_categories": map[string]any{"room_events": map[string]any{"search_term": "pocket"}},
}, token)
if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "stored in a pocket") {
t.Errorf("search: %d %s", rec.Code, rec.Body.String())
}
server, id := uploadThrough(t, srv, token, "text/plain", "note.txt", []byte("pocket media"))
rec = getWithToken(t, srv, "/_matrix/client/v1/media/download/"+server+"/"+id, token)
if rec.Code != http.StatusOK || rec.Body.String() != "pocket media" {
t.Fatalf("download: %d %q", rec.Code, rec.Body.String())
}
var found bool
_ = filepath.WalkDir(filepath.Join(dataDir, "storage", "warp_media"), func(_ string, d os.DirEntry, err error) error {
if err == nil && !d.IsDir() {
found = true
}
return nil
})
if !found {
t.Error("the upload is not in PocketBase's storage directory")
}
if _, err := os.Stat(mediaDir); !os.IsNotExist(err) {
t.Errorf("media.base_path was used: %v", err)
}
}
func TestVersionsAdvertisesSlidingSync(t *testing.T) {
srv := newTestServer(t, nil)
rec := get(t, srv, http.MethodGet, "/_matrix/client/versions")
features := decode[map[string]any](t, rec)["unstable_features"].(map[string]any)
if features["org.matrix.simplified_msc3575"] != true {
t.Errorf("unstable_features = %v", features)
}
}
func TestPhase10RoutesAreWiredUp(t *testing.T) {
srv := newSigningServer(t, func(c *config.Config) {
c.Registration.Enabled = true
c.ClientAPI.RateLimit.Enabled = false
})
token := registerThrough(t, srv, "alice")
rec := post(t, srv, "/_matrix/client/v3/createRoom", map[string]any{
"creation_content": map[string]any{"type": "m.space"},
}, token)
roomID := decode[map[string]string](t, rec)["room_id"]
escaped := url.PathEscape(roomID)
for _, path := range []string{
"/_matrix/client/v1/rooms/" + escaped + "/hierarchy",
"/_matrix/client/v1/rooms/" + escaped + "/threads",
"/_matrix/client/v1/room_summary/" + escaped,
} {
if rec := getWithToken(t, srv, path, token); rec.Code != http.StatusOK {
t.Errorf("%s: %d %s", path, rec.Code, rec.Body.String())
}
}
rec = post(t, srv, "/_matrix/client/unstable/org.matrix.simplified_msc3575/sync", map[string]any{
"lists": map[string]any{"all": map[string]any{"ranges": [][2]int{{0, 5}}}},
}, token)
if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"pos"`) {
t.Errorf("sliding sync: %d %s", rec.Code, rec.Body.String())
}
}
func TestServerMetrics(t *testing.T) {
srv := newSigningServer(t, nil)
token := registerThrough(t, srv, "alice")
if rec := post(t, srv, "/_matrix/client/v3/createRoom", map[string]any{}, token); rec.Code != http.StatusOK {
t.Fatalf("createRoom: %d", rec.Code)
}
body := get(t, srv, http.MethodGet, "/metrics").Body.String()
for _, want := range []string{
`warp_events_persisted_total{kind="state",origin="local"}`,
"warp_sync_waiting 0",
"warp_sync_sliding_connections 0",
"warp_federation_enabled 0",
"go_sql_open_connections",
} {
if !strings.Contains(body, want) {
t.Errorf("/metrics has no %s", want)
}
}
}
// TestIdleSyncLongPolls is the regression test for two ways an idle
// /sync came back at once instead of waiting, which made every client's
// long poll a busy loop: push rules sent again on every sync of a server
// with no account data written yet, and an all-zero token taken for a
// first sync on a server whose streams had not moved at all.
func TestIdleSyncLongPolls(t *testing.T) {
for _, withRoom := range []bool{false, true} {
t.Run(map[bool]string{false: "empty server", true: "with a room"}[withRoom], func(t *testing.T) {
srv := newSigningServer(t, nil)
token := registerThrough(t, srv, "alice")
if withRoom {
if rec := post(t, srv, "/_matrix/client/v3/createRoom", map[string]any{}, token); rec.Code != http.StatusOK {
t.Fatalf("createRoom: %d", rec.Code)
}
}
first := decode[map[string]any](t, getWithToken(t, srv, "/_matrix/client/v3/sync", token))
since, _ := first["next_batch"].(string)
body := decode[map[string]any](t, getWithToken(t, srv, "/_matrix/client/v3/sync?timeout=0&since="+url.QueryEscape(since), token))
accountData, _ := body["account_data"].(map[string]any)
if events, _ := accountData["events"].([]any); len(events) != 0 {
t.Errorf("an idle incremental sync sent account data again: %d events", len(events))
}
start := time.Now()
getWithToken(t, srv, "/_matrix/client/v3/sync?timeout=1000&since="+url.QueryEscape(since), token)
if waited := time.Since(start); waited < 900*time.Millisecond {
t.Errorf("an idle sync returned after %s instead of waiting out its timeout", waited)
}
})
}
}
+12 -3
View File
@@ -6,6 +6,7 @@ import (
"github.com/dariuszjaniak/warp/server/internal/buildinfo"
"github.com/dariuszjaniak/warp/server/internal/httputil"
"github.com/dariuszjaniak/warp/server/internal/tracing"
)
// SpecVersions is what GET /_matrix/client/versions reports.
@@ -20,8 +21,11 @@ import (
var SpecVersions = []string{"v1.13", "v1.14"}
// UnstableFeatures advertises MSC support. Simplified Sliding Sync
// (MSC4186) is Phase 10.
var UnstableFeatures = map[string]bool{}
// (MSC4186) is how Element X and the matrix-rust-sdk find out they can
// skip /sync.
var UnstableFeatures = map[string]bool{
"org.matrix.simplified_msc3575": true,
}
type versionsResponse struct {
Versions []string `json:"versions"`
@@ -51,7 +55,12 @@ type healthResponse struct {
// routes registers every endpoint this server serves.
func (s *Server) routes() *httputil.Router {
r := httputil.NewRouter(s.metrics.Instrument)
// Every route is counted and traced under its pattern. The span is
// inside the metrics wrapper so that what the histogram times is what
// the trace shows.
r := httputil.NewRouter(func(route string, h http.Handler) http.Handler {
return s.metrics.Instrument(route, tracing.Instrument(route, h))
})
r.HandleFunc("GET /_matrix/client/versions", s.handleVersions)
r.HandleFunc("GET /.well-known/matrix/client", s.handleWellKnownClient)
+29 -7
View File
@@ -33,6 +33,7 @@ import (
"github.com/dariuszjaniak/warp/server/internal/storage"
_ "github.com/dariuszjaniak/warp/server/internal/storage/drivers" // link in the storage drivers
"github.com/dariuszjaniak/warp/server/internal/syncapi"
"github.com/dariuszjaniak/warp/server/internal/tracing"
"github.com/dariuszjaniak/warp/server/internal/userapi"
)
@@ -47,6 +48,10 @@ type Server struct {
federation *federation.Gate
signingKey *signing.KeyPair
// stopTracing flushes and stops the tracer provider tracing.Setup
// installed.
stopTracing func(context.Context) error
users *userapi.API
rooms *roomserver.API
sync *syncapi.API
@@ -122,21 +127,29 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error)
s := &Server{cfg: cfg, log: log, metrics: metrics.NewRegistry()}
stopTracing, err := tracing.Setup(ctx, tracing.Config{
Enabled: cfg.Tracing.Enabled,
Endpoint: cfg.Tracing.Endpoint,
SampleRatio: cfg.Tracing.SampleRatio,
ServiceName: cfg.Tracing.ServiceName,
ServiceVersion: buildinfo.Version,
ServerName: cfg.Global.ServerName,
})
if err != nil {
return nil, err
}
s.stopTracing = stopTracing
if opts.Database != nil {
s.db = opts.Database
} else {
db, err := storage.Open(ctx, storage.Config{
Driver: cfg.Database.Driver,
ConnectionString: cfg.Database.ConnectionString,
MaxOpenConns: cfg.Database.MaxOpenConns,
MaxIdleConns: cfg.Database.MaxIdleConns,
ConnMaxLifetime: cfg.Database.ConnMaxLifetime,
}, log)
db, err := storage.Open(ctx, cfg.Database.Storage(), log)
if err != nil {
return nil, err
}
s.db, s.ownsDB = db, true
}
s.registerDBMetrics()
if err := s.prepareSchema(ctx); err != nil {
_ = s.closeDB()
@@ -258,6 +271,7 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error)
notifier := syncapi.NewNotifier()
emit := &fanout{}
emit.add(notifier)
emit.add(s.registerServerMetrics(notifier))
clientTLS := opts.FederationTLS
if clientTLS == nil {
@@ -734,6 +748,14 @@ func (s *Server) Close() error {
// cursors.
s.appservices.Close()
}
if s.stopTracing != nil {
// Spans still in the batch are sent rather than lost.
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := s.stopTracing(ctx); err != nil {
s.log.Warn("could not flush traces", "error", err)
}
}
return s.closeDB()
}
+121
View File
@@ -0,0 +1,121 @@
package setup_test
import (
"net/http"
"net/url"
"testing"
"github.com/dariuszjaniak/warp/server/internal/setup"
)
// threadCounts is the part of a sync the per-thread count test reads.
type threadCounts struct {
NotificationCount int `json:"notification_count"`
HighlightCount int `json:"highlight_count"`
}
type threadSync struct {
Rooms struct {
Join map[string]struct {
UnreadNotifications *threadCounts `json:"unread_notifications"`
UnreadThreadNotifications map[string]threadCounts `json:"unread_thread_notifications"`
} `json:"join"`
} `json:"rooms"`
}
// threadSyncOnce is an initial sync, with the per-thread counts asked for
// or not.
func threadSyncOnce(t *testing.T, srv *setup.Server, token string, byThread bool) threadSync {
t.Helper()
path := "/_matrix/client/v3/sync?timeout=0"
if byThread {
path += "&filter=" + url.QueryEscape(`{"room":{"timeline":{"unread_thread_notifications":true}}}`)
}
rec := getWithToken(t, srv, path, token)
if rec.Code != http.StatusOK {
t.Fatalf("sync: %d %s", rec.Code, rec.Body.String())
}
return decode[threadSync](t, rec)
}
// TestThreadNotificationCountsThroughTheAPI follows notifications into a
// thread and out again: counted under the thread with the filter flag and
// in the room total without it, and read by a thread receipt, a main
// receipt and an unthreaded one, each clearing only its own part.
func TestThreadNotificationCountsThroughTheAPI(t *testing.T) {
srv := newSigningServer(t, nil)
alice := registerThrough(t, srv, "alice")
bob := registerThrough(t, srv, "bob")
roomID := createPublicRoom(t, srv, alice)
mustOK(t, post(t, srv, "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/join", map[string]any{}, bob), "join")
waitForPush(t, srv)
inThread := func(root string) map[string]any {
return map[string]any{"rel_type": "m.thread", "event_id": root}
}
root := sendText(t, srv, bob, roomID, "r", map[string]any{"msgtype": "m.text", "body": "a thread"})
sendText(t, srv, bob, roomID, "a", map[string]any{"msgtype": "m.text", "body": "first reply",
"m.relates_to": inThread(root)})
reply := sendText(t, srv, bob, roomID, "b", map[string]any{"msgtype": "m.text", "body": "second reply",
"m.mentions": map[string]any{"user_ids": []string{"@alice:warp.example"}},
"m.relates_to": inThread(root)})
sendText(t, srv, bob, roomID, "c", map[string]any{"msgtype": "m.text", "body": "back in the room"})
waitForPush(t, srv)
// Without the flag the room counts everything, as before threads.
plain := threadSyncOnce(t, srv, alice, false).Rooms.Join[roomID]
if c := plain.UnreadNotifications; c == nil || *c != (threadCounts{NotificationCount: 4, HighlightCount: 1}) {
t.Errorf("room total = %+v", c)
}
if plain.UnreadThreadNotifications != nil {
t.Errorf("thread counts without the flag: %+v", plain.UnreadThreadNotifications)
}
// With it the root and the later message are the main timeline's, and
// the replies are the thread's.
split := threadSyncOnce(t, srv, alice, true).Rooms.Join[roomID]
if c := split.UnreadNotifications; c == nil || *c != (threadCounts{NotificationCount: 2}) {
t.Errorf("main timeline = %+v", c)
}
if got := split.UnreadThreadNotifications; len(got) != 1 ||
got[root] != (threadCounts{NotificationCount: 2, HighlightCount: 1}) {
t.Errorf("thread counts = %+v", got)
}
receipt := func(eventID string, body map[string]any) {
t.Helper()
mustOK(t, post(t, srv, "/_matrix/client/v3/rooms/"+url.PathEscape(roomID)+"/receipt/m.read/"+url.PathEscape(eventID),
body, alice), "receipt")
}
// Reading the thread leaves the main timeline unread.
receipt(reply, map[string]any{"thread_id": root})
split = threadSyncOnce(t, srv, alice, true).Rooms.Join[roomID]
if c := split.UnreadNotifications; c == nil || c.NotificationCount != 2 || len(split.UnreadThreadNotifications) != 0 {
t.Errorf("after the thread receipt: %+v, %+v", c, split.UnreadThreadNotifications)
}
if c := threadSyncOnce(t, srv, alice, false).Rooms.Join[roomID].UnreadNotifications; c == nil || c.NotificationCount != 2 {
t.Errorf("room total after the thread receipt = %+v", c)
}
// A new reply, then a main receipt on the root: the root is read, the
// later message and the new reply are not.
third := sendText(t, srv, bob, roomID, "d", map[string]any{"msgtype": "m.text", "body": "third reply",
"m.relates_to": inThread(root)})
waitForPush(t, srv)
receipt(root, map[string]any{"thread_id": "main"})
split = threadSyncOnce(t, srv, alice, true).Rooms.Join[roomID]
if c := split.UnreadNotifications; c == nil || c.NotificationCount != 1 ||
split.UnreadThreadNotifications[root].NotificationCount != 1 {
t.Errorf("after the main receipt: %+v, %+v", c, split.UnreadThreadNotifications)
}
// An unthreaded receipt reads the lot up to its event, whichever
// thread that event is in: here the newest, a reply.
receipt(third, map[string]any{})
split = threadSyncOnce(t, srv, alice, true).Rooms.Join[roomID]
if c := split.UnreadNotifications; c == nil || c.NotificationCount != 0 || len(split.UnreadThreadNotifications) != 0 {
t.Errorf("after the unthreaded receipt: %+v, %+v", c, split.UnreadThreadNotifications)
}
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,77 @@
-- Phase 10: relations (threads, edits, reactions, references) and
-- full-text search.
-- content.m.relates_to of each event, written out so that "the children of
-- this event" is an index lookup. Both are '' for an event that relates to
-- nothing, and are cleared when the event is redacted: the redaction
-- algorithm strips the content they came from.
ALTER TABLE roomserver_events ADD COLUMN relates_to_id TEXT NOT NULL DEFAULT '';
ALTER TABLE roomserver_events ADD COLUMN rel_type TEXT NOT NULL DEFAULT '';
-- The events already stored. Only a relation with both fields a string
-- counts, which is the rule the roomserver applies to new events. The LIKE
-- keeps the cast to jsonb off the events that cannot have one.
UPDATE roomserver_events e SET
relates_to_id = r.rel ->> 'event_id',
rel_type = r.rel ->> 'rel_type'
FROM (
SELECT event_id, event_json::jsonb -> 'content' -> 'm.relates_to' AS rel
FROM roomserver_events
WHERE redacted_by = '' AND event_json LIKE '%m.relates_to%'
) r
WHERE e.event_id = r.event_id
AND jsonb_typeof(r.rel -> 'event_id') = 'string'
AND jsonb_typeof(r.rel -> 'rel_type') = 'string';
-- /relations, the bundled summaries and /threads all start from a parent
-- and read its children in stream order.
CREATE INDEX IF NOT EXISTS roomserver_events_relations_idx
ON roomserver_events (room_id, relates_to_id, rel_type, stream_ordering);
-- The full-text index /search reads: one row per searchable event, keyed by
-- the event's stream ordering, which is unique across the server and is
-- what a redaction finds the row by. Outliers are not here: an event joins
-- the index when it joins a timeline.
--
-- The 'simple' configuration lower-cases and splits on word boundaries
-- without stemming for any one language, which is the right default for a
-- server whose rooms are in every language at once.
CREATE TABLE IF NOT EXISTS roomserver_event_search (
stream_ordering BIGINT NOT NULL PRIMARY KEY,
room_id TEXT NOT NULL,
event_id TEXT NOT NULL,
search_key TEXT NOT NULL,
sender TEXT NOT NULL,
event_type TEXT NOT NULL,
body TEXT NOT NULL,
vector TSVECTOR GENERATED ALWAYS AS (to_tsvector('simple', body)) STORED
);
CREATE INDEX IF NOT EXISTS roomserver_event_search_vector_idx
ON roomserver_event_search USING GIN (vector);
CREATE INDEX IF NOT EXISTS roomserver_event_search_room_idx
ON roomserver_event_search (room_id, stream_ordering);
INSERT INTO roomserver_event_search (stream_ordering, room_id, event_id, search_key, sender, event_type, body)
SELECT stream_ordering, room_id, event_id, k.search_key, sender, event_type, k.body
FROM (
SELECT stream_ordering, room_id, event_id, sender, event_type, event_json::jsonb -> 'content' AS content
FROM roomserver_events
WHERE event_type IN ('m.room.message', 'm.room.name', 'm.room.topic')
AND NOT outlier AND redacted_by = ''
) e
CROSS JOIN LATERAL (
SELECT CASE e.event_type
WHEN 'm.room.message' THEN 'content.body'
WHEN 'm.room.name' THEN 'content.name'
ELSE 'content.topic'
END AS search_key,
CASE e.event_type
WHEN 'm.room.message' THEN e.content ->> 'body'
WHEN 'm.room.name' THEN e.content ->> 'name'
ELSE e.content ->> 'topic'
END AS body
) k
WHERE k.body IS NOT NULL
AND jsonb_typeof(e.content -> substr(k.search_key, 9)) = 'string'
ON CONFLICT (stream_ordering) DO NOTHING;
@@ -0,0 +1,21 @@
-- Phase 10: notifications counted per thread.
--
-- thread_id is the thread an event that notified a user belongs to: the
-- root its m.relates_to names when rel_type is m.thread, and '' for an
-- event on the main timeline. A read receipt for a thread clears that
-- thread's rows and a receipt for the main timeline clears the '' rows,
-- which is what lets reading one thread leave the rest of the room unread.
ALTER TABLE push_notifications ADD COLUMN thread_id TEXT NOT NULL DEFAULT '';
-- The notifications already stored, from the relation 0012 projected onto
-- their events. A notification whose event has gone stays on the main
-- timeline, where an unthreaded or main receipt still clears it.
UPDATE push_notifications SET thread_id = COALESCE((
SELECT e.relates_to_id FROM roomserver_events e
WHERE e.event_id = push_notifications.event_id AND e.rel_type = 'm.thread'
), '');
-- A threaded receipt marks one thread's rows read; the per-thread counts a
-- sync asks for group a room's unread rows by thread.
CREATE INDEX IF NOT EXISTS push_notifications_thread_idx
ON push_notifications (user_id, room_id, thread_id, is_read, event_pos);
@@ -9,6 +9,7 @@ import (
"embed"
"fmt"
"log/slog"
"strings"
_ "github.com/jackc/pgx/v5/stdlib" // registers the "pgx" database/sql driver
@@ -44,6 +45,33 @@ var settingsQueries = sqlstore.SettingsQueries{
Delete: `DELETE FROM server_settings WHERE setting_key = $1`,
}
// searchDialect is the full-text index on tsvector, with the 'simple'
// configuration the migration builds the column with.
var searchDialect = &sqlstore.SearchDialect{
Insert: `INSERT INTO roomserver_event_search (stream_ordering, room_id, event_id, search_key, sender, event_type, body)
VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT (stream_ordering) DO NOTHING`,
DeleteEvent: `DELETE FROM roomserver_event_search WHERE room_id = ? AND event_id = ?`,
DeleteRoom: `DELETE FROM roomserver_event_search WHERE room_id = ?`,
From: `roomserver_event_search s`,
Name: `s`,
Position: `s.stream_ordering`,
Match: `s.vector @@ to_tsquery('simple', ?)`,
Rank: `ts_rank(s.vector, to_tsquery('simple', ?))`,
RankArgs: 1,
Query: tsQuery,
}
// tsQuery builds a tsquery from a search term: every word as a prefix
// match, ANDed. The words are letters and digits only, so none of them can
// carry tsquery syntax of its own.
func tsQuery(term string) string {
words := storage.SearchWords(term)
for i, w := range words {
words[i] = w + ":*"
}
return strings.Join(words, " & ")
}
// Database is the PostgreSQL implementation of storage.Database.
type Database struct {
*sqlstore.Store
@@ -77,6 +105,7 @@ func Open(ctx context.Context, cfg storage.Config, log *slog.Logger) (storage.Da
Settings: settingsQueries,
Rebind: rebind,
IsUniqueViolation: isUniqueViolation,
Search: searchDialect,
Logger: log,
})
if err != nil {
+29 -2
View File
@@ -47,6 +47,11 @@ type NewNotification struct {
// what a read receipt is compared against.
EventPos int64
// ThreadID is the thread the event is in: the root its m.relates_to
// names when rel_type is m.thread, or ThreadMain for an event on the
// main timeline. A thread root is itself on the main timeline.
ThreadID string
TimestampMS int64
Highlight bool
@@ -66,6 +71,19 @@ type Notification struct {
Read bool
}
// Threads a read receipt can be for, as the thread argument of
// MarkNotificationsRead. Any other value is a thread root's event ID.
const (
// ThreadMain is the main timeline: the thread ID of an event in no
// thread, and what a receipt with thread_id "main" reads.
ThreadMain = ""
// AllThreads is an unthreaded receipt, which reads the main timeline
// and every thread together. It cannot be mistaken for a thread root:
// an event ID starts with '$'.
AllThreads = "*"
)
// NotificationCounts is a room's unread notifications for one user.
type NotificationCounts struct {
Notifications int
@@ -105,13 +123,22 @@ type PushStore interface {
// MarkNotificationsRead marks a user's notifications in a room up to
// and including an event position as read, and reports whether the
// counts moved.
MarkNotificationsRead(ctx context.Context, userID, roomID string, upTo int64) (pos int64, moved bool, err error)
// counts moved. thread limits it to one thread — ThreadMain for the
// main timeline, a root's event ID for a thread — or is AllThreads.
MarkNotificationsRead(ctx context.Context, userID, roomID, thread string, upTo int64) (pos int64, moved bool, err error)
// NotificationCounts reads a user's unread counts in the given rooms.
// A room with nothing unread is in the result with zero counts.
NotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]NotificationCounts, error)
// ThreadNotificationCounts reads a user's unread counts in the given
// rooms one thread at a time, keyed by room and then by thread ID, with
// the main timeline under ThreadMain. Only threads with something
// unread are in the result. The room's move in the notification stream
// covers its threads: any change to a thread's counts is a change to
// the room's.
ThreadNotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]map[string]NotificationCounts, error)
// NotificationCountChanges lists the rooms whose counts for a user
// moved after from and at or before to.
NotificationCountChanges(ctx context.Context, userID string, from, to int64) ([]string, error)
+99 -12
View File
@@ -2,7 +2,6 @@ package storage_test
import (
"database/sql"
"errors"
"os"
"slices"
"testing"
@@ -151,7 +150,7 @@ func TestNotifications(t *testing.T) {
// Reading up to the first event clears it alone, and moves the
// stream for that room only.
pos, moved, err := push.MarkNotificationsRead(ctx, alice, room1, 10)
pos, moved, err := push.MarkNotificationsRead(ctx, alice, room1, storage.AllThreads, 10)
if err != nil || !moved || pos != last+1 {
t.Fatalf("MarkNotificationsRead = %d, %v, %v", pos, moved, err)
}
@@ -164,11 +163,11 @@ func TestNotifications(t *testing.T) {
}
// Reading the same again moves nothing.
if again, moved, err := push.MarkNotificationsRead(ctx, alice, room1, 10); err != nil || moved || again != pos {
if again, moved, err := push.MarkNotificationsRead(ctx, alice, room1, storage.AllThreads, 10); err != nil || moved || again != pos {
t.Errorf("reading again = %d, %v, %v", again, moved, err)
}
if _, _, err := push.MarkNotificationsRead(ctx, alice, room1, 100); err != nil {
if _, _, err := push.MarkNotificationsRead(ctx, alice, room1, storage.AllThreads, 100); err != nil {
t.Fatal(err)
}
counts, _ = push.NotificationCounts(ctx, alice, []string{room1})
@@ -206,13 +205,101 @@ func TestNotifications(t *testing.T) {
}
}
func TestPushNotImplementedOnPocketBase(t *testing.T) {
db, err := storage.Open(t.Context(), storage.Config{Driver: storage.DriverPocketBase}, nil)
if err != nil {
t.Skipf("pocketbase: %v", err)
}
defer func() { _ = db.Close() }()
if _, err := db.Push().Pushers(t.Context(), "alice"); !errors.Is(err, storage.ErrNotImplemented) {
t.Errorf("Pushers = %v", err)
// TestThreadNotifications covers notifications filed by thread: a receipt
// for one thread, for the main timeline, and for the whole room each read
// only their own part, while the room totals stay the sum of every part.
func TestThreadNotifications(t *testing.T) {
const alice = "@alice:warp.example"
const room, other = "!one:warp.example", "!two:warp.example"
const t1, t2 = "$root1", "$root2"
for _, driver := range testDrivers {
t.Run(driver, func(t *testing.T) {
ctx := t.Context()
push := migratedPushDB(t, driver).Push()
note := func(event, thread string, pos int64, highlight bool) storage.NewNotification {
return storage.NewNotification{UserID: alice, RoomID: room, EventID: event, EventPos: pos,
ThreadID: thread, TimestampMS: 1000 + pos, Highlight: highlight, Actions: []byte(`["notify"]`)}
}
if _, err := push.AddNotifications(ctx, []storage.NewNotification{
note(t1, storage.ThreadMain, 10, false), // a thread root is on the main timeline
note("$a", t1, 11, true),
note("$b", t1, 12, false),
note("$c", t2, 13, false),
note("$d", storage.ThreadMain, 14, false),
}); err != nil {
t.Fatal(err)
}
byThread := func() map[string]storage.NotificationCounts {
t.Helper()
counts, err := push.ThreadNotificationCounts(ctx, alice, []string{room, other})
if err != nil {
t.Fatal(err)
}
if _, ok := counts[other]; ok {
t.Errorf("a room with nothing unread is in the result: %+v", counts[other])
}
return counts[room]
}
total := func() storage.NotificationCounts {
t.Helper()
counts, err := push.NotificationCounts(ctx, alice, []string{room})
if err != nil {
t.Fatal(err)
}
return counts[room]
}
got := byThread()
if len(got) != 3 ||
got[storage.ThreadMain] != (storage.NotificationCounts{Notifications: 2}) ||
got[t1] != (storage.NotificationCounts{Notifications: 2, Highlights: 1}) ||
got[t2] != (storage.NotificationCounts{Notifications: 1}) {
t.Errorf("ThreadNotificationCounts = %+v", got)
}
if all, _ := push.Notifications(ctx, alice, 0, 10, false); len(all) != 5 || all[1].ThreadID != t2 {
t.Errorf("Notifications = %+v", all)
}
// A receipt in thread 1 reads that thread, past everything else
// in the room, and leaves the rest unread.
if _, moved, err := push.MarkNotificationsRead(ctx, alice, room, t1, 100); err != nil || !moved {
t.Fatalf("reading thread 1 = %v, %v", moved, err)
}
got = byThread()
if _, ok := got[t1]; ok || len(got) != 2 {
t.Errorf("after reading thread 1: %+v", got)
}
if c := total(); c != (storage.NotificationCounts{Notifications: 3}) {
t.Errorf("the room total after reading thread 1 = %+v", c)
}
// A main receipt reads the main timeline up to its event.
if _, moved, err := push.MarkNotificationsRead(ctx, alice, room, storage.ThreadMain, 10); err != nil || !moved {
t.Fatalf("reading the main timeline = %v, %v", moved, err)
}
got = byThread()
if got[storage.ThreadMain] != (storage.NotificationCounts{Notifications: 1}) ||
got[t2] != (storage.NotificationCounts{Notifications: 1}) {
t.Errorf("after the main receipt: %+v", got)
}
// A receipt for a thread with nothing unread moves nothing.
if _, moved, err := push.MarkNotificationsRead(ctx, alice, room, t1, 100); err != nil || moved {
t.Errorf("reading thread 1 again = %v, %v", moved, err)
}
// An unthreaded receipt reads everything.
if _, _, err := push.MarkNotificationsRead(ctx, alice, room, storage.AllThreads, 100); err != nil {
t.Fatal(err)
}
if got := byThread(); len(got) != 0 {
t.Errorf("after the unthreaded receipt: %+v", got)
}
if c := total(); c != (storage.NotificationCounts{}) {
t.Errorf("the room total after the unthreaded receipt = %+v", c)
}
})
}
}
+189
View File
@@ -0,0 +1,189 @@
package storage
import "context"
// The relation and search reads of the room store: threads, edits,
// reactions and references (an event's content.m.relates_to), and the
// full-text index /search is answered from.
//
// Both are projections of the event JSON that the roomserver works out and
// hands over with the event, the same as its type and state key: the store
// still never parses an event. They live in a file of their own because
// they are the Phase 10 additions to the room store, and because neither is
// about a room's state or its stream order — they are about which events
// point at which, and which events say what.
// Relation types the server aggregates. Every rel_type is recorded and can
// be listed through /relations; these are the ones whose summaries go into
// an event's unsigned.m.relations.
const (
RelThread = "m.thread"
RelReplace = "m.replace"
RelReference = "m.reference"
RelAnnotate = "m.annotation"
)
// RelationsRequest is one page of the events relating to some parents.
//
// The bounds are a TimelineRequest's: Low exclusive, High inclusive, zero
// unbounded at either end, positions possibly negative inside backfilled
// history.
type RelationsRequest struct {
RoomID string
// ParentIDs are the events whose children are read. It is one event
// for a plain /relations call and the event plus its descendants for a
// recursive one.
ParentIDs []string
// RelType and EventType narrow the children, when set.
RelType string
EventType string
Low int64
High int64
Direction TimelineDirection
// Limit caps the rows returned, and is required.
Limit int
}
// RelationAggregate is what one parent's children of one relation type
// come to, from one sender.
//
// It is per sender because two of the summaries need it: an edit only
// counts when it comes from the original's sender, and a thread reports
// whether the user asking took part. Summing over senders is then the
// roomserver's job.
type RelationAggregate struct {
ParentID string
RelType string
Sender string
// Count is how many children there are.
Count int
// LatestEventID is the newest of them, by stream order.
LatestEventID string
// LatestOrdering is its position.
LatestOrdering int64
}
// ThreadsRequest is one page of a room's threads, newest activity first.
type ThreadsRequest struct {
RoomID string
// ParticipatedBy, when set, keeps only the threads that user started
// or replied in.
ParticipatedBy string
// Before is an exclusive bound on a thread's latest activity: the page
// after one whose last thread was last active at Before. Zero is the
// first page.
Before int64
// High bounds what counts as activity, inclusive, for a user whose
// view of the room ends at their leave. Zero is unbounded.
High int64
// Limit caps the threads returned, and is required.
Limit int
}
// Thread is one thread of a room.
type Thread struct {
RootID string
// LatestOrdering is the position of the newest event in the thread,
// which is what the listing is ordered and paged by.
LatestOrdering int64
}
// SearchOrder is how /search orders its results.
type SearchOrder string
// The two orders the Client-Server API defines.
const (
SearchByRank SearchOrder = "rank"
SearchByRecent SearchOrder = "recent"
)
// SearchRange is the part of one room a user may search: the window of its
// stream their membership and the room's history visibility allow. The
// bounds are a TimelineRequest's.
type SearchRange struct {
RoomID string
Low int64
High int64
}
// SearchRequest is one page of a full-text search.
type SearchRequest struct {
// Term is what the user typed. The store turns it into its engine's
// query syntax; a term with no searchable words matches nothing.
Term string
// Rooms are where to look, each bounded to what the user may see.
Rooms []SearchRange
// Keys narrows which fields match: "content.body", "content.name",
// "content.topic". Empty means all three.
Keys []string
// Filter narrows by type and sender, as a timeline filter does.
Filter TimelineFilter
Order SearchOrder
// Offset skips results, for paging a ranked search. A search by
// recency pages by Before instead, which stays put as events arrive.
Offset int
// Before is an exclusive upper bound on the stream position, for
// paging a search by recency. Zero is the first page.
Before int64
// Limit caps the results returned, and is required.
Limit int
}
// SearchResult is one event a search found.
type SearchResult struct {
Event StoredEvent
// Rank is the engine's relevance score, higher is better. It is only
// comparable within one search.
Rank float64
}
// RelationStore is the part of the room store about relations and search.
// It is embedded in RoomStore; it is a separate interface only so that the
// two halves can be read apart.
type RelationStore interface {
// Relations reads one page of the children of the given parents, in
// stream order in the direction asked for.
Relations(ctx context.Context, req RelationsRequest) ([]StoredEvent, error)
// ChildEventIDs lists the direct children of the given parents,
// narrowed by relation and event type when those are set. It is how a
// recursive /relations walks down a level at a time.
ChildEventIDs(ctx context.Context, roomID string, parentIDs []string, relType, eventType string) ([]string, error)
// RelationAggregates summarises the thread, edit and reference
// children of the given parents, up to and including high (zero for
// no bound).
RelationAggregates(ctx context.Context, roomID string, parentIDs []string, high int64) ([]RelationAggregate, error)
// RelationEventIDs lists the children of one relation type of the
// given parents, oldest first, at most limit of them: the chunk of an
// m.reference summary.
RelationEventIDs(ctx context.Context, roomID string, parentIDs []string, relType string, high int64, limit int) (map[string][]string, error)
// Threads reads one page of a room's threads.
Threads(ctx context.Context, req ThreadsRequest) ([]Thread, error)
// Search reads one page of a full-text search and the total number of
// matches.
Search(ctx context.Context, req SearchRequest) ([]SearchResult, int, error)
}
+279
View File
@@ -0,0 +1,279 @@
package storage_test
import (
"slices"
"testing"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// writeMessage appends a timeline event with the given projections and
// returns its stream ordering.
func writeMessage(t *testing.T, rooms storage.RoomStore, ev storage.StoredEvent) int64 {
t.Helper()
pos, err := rooms.Write(t.Context(), storage.RoomWrite{Event: ev, Extremities: []string{ev.EventID}})
if err != nil {
t.Fatalf("Write %s: %v", ev.EventID, err)
}
return pos
}
func relationEvent(id, sender, parent, relType string) storage.StoredEvent {
e := newTestEvent(testRoomID, id, "m.room.message", nil)
e.Sender = sender
e.RelatesTo, e.RelType = parent, relType
return e
}
func TestRelationsAndAggregates(t *testing.T) {
for _, driver := range testDrivers {
t.Run(driver, func(t *testing.T) {
ctx := t.Context()
rooms := migratedRoomDB(t, driver).Rooms()
if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil {
t.Fatalf("CreateRoom: %v", err)
}
writeMessage(t, rooms, newTestEvent(testRoomID, "$root", "m.room.message", nil))
writeMessage(t, rooms, relationEvent("$t1", "@bob:warp.example", "$root", storage.RelThread))
last := writeMessage(t, rooms, relationEvent("$t2", "@carol:warp.example", "$root", storage.RelThread))
writeMessage(t, rooms, relationEvent("$edit", "@alice:warp.example", "$root", storage.RelReplace))
writeMessage(t, rooms, relationEvent("$ref", "@bob:warp.example", "$root", storage.RelReference))
writeMessage(t, rooms, relationEvent("$t1edit", "@bob:warp.example", "$t1", storage.RelReplace))
children, err := rooms.Relations(ctx, storage.RelationsRequest{
RoomID: testRoomID, ParentIDs: []string{"$root"},
Direction: storage.DirectionBackward, Limit: 10,
})
if err != nil {
t.Fatalf("Relations: %v", err)
}
if got := eventIDs(children); !slices.Equal(got, []string{"$ref", "$edit", "$t2", "$t1"}) {
t.Errorf("Relations = %v", got)
}
if children[0].RelatesTo != "$root" || children[0].RelType != storage.RelReference {
t.Errorf("the projections did not round-trip: %+v", children[0])
}
threads, err := rooms.Relations(ctx, storage.RelationsRequest{
RoomID: testRoomID, ParentIDs: []string{"$root", "$t1"}, RelType: storage.RelReplace,
Direction: storage.DirectionForward, Limit: 10,
})
if err != nil {
t.Fatalf("Relations by type: %v", err)
}
if got := eventIDs(threads); !slices.Equal(got, []string{"$edit", "$t1edit"}) {
t.Errorf("m.replace children of two parents = %v", got)
}
ids, err := rooms.ChildEventIDs(ctx, testRoomID, []string{"$t1"}, "", "")
if err != nil || !slices.Equal(ids, []string{"$t1edit"}) {
t.Errorf("ChildEventIDs = %v, %v", ids, err)
}
aggregates, err := rooms.RelationAggregates(ctx, testRoomID, []string{"$root"}, 0)
if err != nil {
t.Fatalf("RelationAggregates: %v", err)
}
threadCount, latest := 0, ""
for _, a := range aggregates {
if a.RelType == storage.RelThread {
threadCount += a.Count
if a.LatestOrdering == last {
latest = a.LatestEventID
}
}
}
if threadCount != 2 || latest != "$t2" || len(aggregates) != 4 {
t.Errorf("aggregates = %+v", aggregates)
}
refs, err := rooms.RelationEventIDs(ctx, testRoomID, []string{"$root"}, storage.RelReference, 0, 10)
if err != nil || !slices.Equal(refs["$root"], []string{"$ref"}) {
t.Errorf("RelationEventIDs = %v, %v", refs, err)
}
// Bounded above, the later replies are not counted.
bounded, err := rooms.RelationAggregates(ctx, testRoomID, []string{"$root"}, last-1)
if err != nil {
t.Fatalf("RelationAggregates bounded: %v", err)
}
for _, a := range bounded {
if a.RelType == storage.RelThread && a.LatestEventID == "$t2" {
t.Errorf("a bounded aggregate saw the event above its bound: %+v", a)
}
}
})
}
}
func TestThreadsListing(t *testing.T) {
for _, driver := range testDrivers {
t.Run(driver, func(t *testing.T) {
ctx := t.Context()
rooms := migratedRoomDB(t, driver).Rooms()
if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil {
t.Fatalf("CreateRoom: %v", err)
}
writeMessage(t, rooms, newTestEvent(testRoomID, "$a", "m.room.message", nil))
b := newTestEvent(testRoomID, "$b", "m.room.message", nil)
b.Sender = "@bob:warp.example"
writeMessage(t, rooms, b)
writeMessage(t, rooms, relationEvent("$a1", "@alice:warp.example", "$a", storage.RelThread))
writeMessage(t, rooms, relationEvent("$b1", "@alice:warp.example", "$b", storage.RelThread))
// A thread whose root this server does not hold is not listed.
writeMessage(t, rooms, relationEvent("$x1", "@alice:warp.example", "$missing", storage.RelThread))
all, err := rooms.Threads(ctx, storage.ThreadsRequest{RoomID: testRoomID, Limit: 10})
if err != nil {
t.Fatalf("Threads: %v", err)
}
if got := threadRoots(all); !slices.Equal(got, []string{"$b", "$a"}) {
t.Errorf("Threads = %v, want newest activity first", got)
}
page, err := rooms.Threads(ctx, storage.ThreadsRequest{RoomID: testRoomID, Limit: 1, Before: all[0].LatestOrdering})
if err != nil || !slices.Equal(threadRoots(page), []string{"$a"}) {
t.Errorf("Threads after the first = %v, %v", threadRoots(page), err)
}
bobs, err := rooms.Threads(ctx, storage.ThreadsRequest{
RoomID: testRoomID, ParticipatedBy: "@bob:warp.example", Limit: 10,
})
if err != nil || !slices.Equal(threadRoots(bobs), []string{"$b"}) {
t.Errorf("Threads bob took part in = %v, %v", threadRoots(bobs), err)
}
})
}
}
func TestSearch(t *testing.T) {
for _, driver := range testDrivers {
t.Run(driver, func(t *testing.T) {
ctx := t.Context()
rooms := migratedRoomDB(t, driver).Rooms()
const other = "!other:warp.example"
for _, id := range []string{testRoomID, other} {
if err := rooms.CreateRoom(ctx, newTestRoom(id)); err != nil {
t.Fatalf("CreateRoom: %v", err)
}
}
message := func(roomID, id, body string) int64 {
e := newTestEvent(roomID, id, "m.room.message", nil)
e.SearchKey, e.SearchText = "content.body", body
return writeMessage(t, rooms, e)
}
message(testRoomID, "$hello", "Hello wonderful world")
message(testRoomID, "$cafe", "Meet at the café tomorrow")
second := message(testRoomID, "$hello2", "hello again, hello")
message(other, "$elsewhere", "hello from elsewhere")
topic := newTestEvent(testRoomID, "$topic", "m.room.topic", stateKey(""))
topic.SearchKey, topic.SearchText = "content.topic", "a room to say hello in"
writeMessage(t, rooms, topic)
search := func(req storage.SearchRequest) ([]string, int) {
t.Helper()
if req.Limit == 0 {
req.Limit = 10
}
if req.Rooms == nil {
req.Rooms = []storage.SearchRange{{RoomID: testRoomID}}
}
results, total, err := rooms.Search(ctx, req)
if err != nil {
t.Fatalf("Search(%q): %v", req.Term, err)
}
out := make([]string, 0, len(results))
for _, r := range results {
out = append(out, r.Event.EventID)
}
return out, total
}
got, total := search(storage.SearchRequest{Term: "hello", Order: storage.SearchByRecent})
if !slices.Equal(got, []string{"$topic", "$hello2", "$hello"}) || total != 3 {
t.Errorf("recent search for hello = %v (total %d)", got, total)
}
// A prefix matches, and diacritics are folded on SQLite; the
// tsvector 'simple' configuration keeps them, so the term is
// spelled as stored.
if got, _ := search(storage.SearchRequest{Term: "hel"}); len(got) != 3 {
t.Errorf("prefix search = %v", got)
}
if got, _ := search(storage.SearchRequest{Term: "café"}); !slices.Equal(got, []string{"$cafe"}) {
t.Errorf("search for café = %v", got)
}
// Operators in the term are words, not syntax.
if got, _ := search(storage.SearchRequest{Term: `"hello" & (world`}); !slices.Equal(got, []string{"$hello"}) {
t.Errorf("a term full of operators = %v, want the one event with both words", got)
}
if got, _ := search(storage.SearchRequest{Term: "!!!"}); len(got) != 0 {
t.Errorf("a term with no words found %v", got)
}
if got, _ := search(storage.SearchRequest{Term: "hello", Keys: []string{"content.topic"}}); !slices.Equal(got, []string{"$topic"}) {
t.Errorf("search by key = %v", got)
}
if got, _ := search(storage.SearchRequest{
Term: "hello", Rooms: []storage.SearchRange{{RoomID: testRoomID, High: second - 1}},
}); !slices.Equal(got, []string{"$hello"}) {
t.Errorf("search bounded below the second hello = %v", got)
}
if got, total := search(storage.SearchRequest{
Term: "hello", Rooms: []storage.SearchRange{{RoomID: testRoomID}, {RoomID: other}},
}); len(got) != 4 || total != 4 {
t.Errorf("search across two rooms = %v (total %d)", got, total)
}
if got, _ := search(storage.SearchRequest{
Term: "hello", Filter: storage.TimelineFilter{NotTypes: []string{"m.room.topic"}}, Order: storage.SearchByRecent,
}); !slices.Equal(got, []string{"$hello2", "$hello"}) {
t.Errorf("filtered search = %v", got)
}
// Paging by recency, then by rank.
page1, _ := search(storage.SearchRequest{Term: "hello", Order: storage.SearchByRecent, Limit: 2})
if len(page1) != 2 {
t.Fatalf("first page = %v", page1)
}
if page2, _ := search(storage.SearchRequest{Term: "hello", Order: storage.SearchByRank, Offset: 2, Limit: 2}); len(page2) != 1 {
t.Errorf("ranked second page = %v", page2)
}
// A redacted event is no longer found by what it said.
redaction := newTestEvent(testRoomID, "$redaction", "m.room.redaction", nil)
if _, err := rooms.Write(ctx, storage.RoomWrite{
Event: redaction, Extremities: []string{"$redaction"},
Redacts: "$hello2", RedactedJSON: []byte(`{"type":"m.room.message"}`),
}); err != nil {
t.Fatalf("Write redaction: %v", err)
}
if got, _ := search(storage.SearchRequest{Term: "again"}); len(got) != 0 {
t.Errorf("the redacted event is still found: %v", got)
}
if err := rooms.PurgeRoom(ctx, testRoomID); err != nil {
t.Fatalf("PurgeRoom: %v", err)
}
if got, _ := search(storage.SearchRequest{Term: "hello"}); len(got) != 0 {
t.Errorf("a purged room is still searched: %v", got)
}
})
}
}
func eventIDs(events []storage.StoredEvent) []string {
out := make([]string, 0, len(events))
for _, e := range events {
out = append(out, e.EventID)
}
return out
}
func threadRoots(threads []storage.Thread) []string {
out := make([]string, 0, len(threads))
for _, t := range threads {
out = append(out, t.RootID)
}
return out
}
+25
View File
@@ -140,6 +140,22 @@ type StoredEvent struct {
// soft-failed. An outlier can be cited and read by ID, and is never in
// a timeline.
Outlier bool
// RelatesTo and RelType are content.m.relates_to's event_id and
// rel_type, empty for an event that relates to nothing. A reply's
// m.in_reply_to is not a relation in this sense and is not recorded.
// They are cleared when the event is redacted, because the redaction
// algorithm strips the content they came from.
RelatesTo string
RelType string
// SearchKey and SearchText are what the full-text index holds for the
// event: the field ("content.body", "content.name" or "content.topic")
// and its text. Both are empty for an event with nothing to search.
// They are written, never read back: the index is the only place they
// are kept.
SearchKey string
SearchText string
}
// IsState reports whether the event is a state event.
@@ -302,6 +318,8 @@ type RoomStats struct {
// guarantees is that a room's events, its state and its extremities are only
// ever seen in a consistent set.
type RoomStore interface {
RelationStore
// CreateRoom inserts the room row, returning ErrRoomExists when the ID
// is taken.
CreateRoom(ctx context.Context, room NewRoom) error
@@ -398,6 +416,13 @@ type RoomStore interface {
// otherwise need two hundred timeline reads to find that out.
RoomsWithEvents(ctx context.Context, roomIDs []string, from, to int64) ([]string, error)
// LatestEventPositions returns, for each of roomIDs that has one, the
// position of its newest timeline event of one of the given types at
// or before upTo (zero for no bound). It is what sliding sync orders a
// room list by: the last thing that happened in each room that a user
// would call activity.
LatestEventPositions(ctx context.Context, roomIDs, types []string, upTo int64) (map[string]int64, error)
// StateAtStream returns the room's state as of a stream position: the
// state after the last event at or before pos. The position may be a
// negative one inside backfilled history; zero has no events at or
+1 -1
View File
@@ -42,7 +42,7 @@ func resetRoomTables(t *testing.T, driver string) {
roomserver_state_entries, roomserver_state_snapshots,
roomserver_current_state, roomserver_extremities,
roomserver_transactions, roomserver_aliases, roomserver_invite_state,
roomserver_events, roomserver_rooms`); err != nil {
roomserver_events, roomserver_rooms, roomserver_event_search`); err != nil {
t.Fatalf("reset the room tables: %v", err)
}
if _, err := db.ExecContext(t.Context(),
+26
View File
@@ -0,0 +1,26 @@
package storage
import (
"strings"
"unicode"
)
// SearchWords splits what a user typed into a search box into the words
// an index can match: runs of letters and digits, lower-cased, everything
// else a separator.
//
// Both engines' query languages have operators — FTS5's quotes, AND, NEAR
// and column filters, tsquery's & | ! and parentheses — and a search box is
// not the place to expose them: a user searching for "C++ & Go" means the
// words, not a malformed query. Reducing the term to words and building the
// query from those is what makes any input safe to pass on.
func SearchWords(term string) []string {
fields := strings.FieldsFunc(term, func(r rune) bool {
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
})
out := fields[:0]
for _, f := range fields {
out = append(out, strings.ToLower(f))
}
return out
}
@@ -0,0 +1,61 @@
-- Phase 10: relations (threads, edits, reactions, references) and
-- full-text search.
-- content.m.relates_to of each event, written out so that "the children of
-- this event" is an index lookup. Both are '' for an event that relates to
-- nothing, and are cleared when the event is redacted: the redaction
-- algorithm strips the content they came from.
ALTER TABLE roomserver_events ADD COLUMN relates_to_id TEXT NOT NULL DEFAULT '';
ALTER TABLE roomserver_events ADD COLUMN rel_type TEXT NOT NULL DEFAULT '';
-- The events already stored. Only a relation with both fields a string
-- counts, which is the rule the roomserver applies to new events.
UPDATE roomserver_events SET
relates_to_id = json_extract(event_json, '$.content."m.relates_to".event_id'),
rel_type = json_extract(event_json, '$.content."m.relates_to".rel_type')
WHERE redacted_by = ''
AND json_type(event_json, '$.content."m.relates_to".event_id') = 'text'
AND json_type(event_json, '$.content."m.relates_to".rel_type') = 'text';
-- /relations, the bundled summaries and /threads all start from a parent
-- and read its children in stream order.
CREATE INDEX IF NOT EXISTS roomserver_events_relations_idx
ON roomserver_events (room_id, relates_to_id, rel_type, stream_ordering);
-- The full-text index /search reads: one row per searchable event, keyed by
-- the event's stream ordering as its rowid, which is unique across the
-- server and is what a redaction finds the row by. Outliers are not here:
-- an event joins the index when it joins a timeline.
--
-- unicode61 with diacritics removed matches "cafe" against "café", which is
-- what a user searching a chat history expects.
CREATE VIRTUAL TABLE IF NOT EXISTS roomserver_event_search USING fts5(
body,
room_id UNINDEXED,
event_id UNINDEXED,
search_key UNINDEXED,
sender UNINDEXED,
event_type UNINDEXED,
tokenize = 'unicode61 remove_diacritics 2'
);
INSERT INTO roomserver_event_search (rowid, body, room_id, event_id, search_key, sender, event_type)
SELECT stream_ordering, json_extract(event_json, '$.content.body'), room_id, event_id,
'content.body', sender, event_type
FROM roomserver_events
WHERE event_type = 'm.room.message' AND NOT outlier AND redacted_by = ''
AND json_type(event_json, '$.content.body') = 'text';
INSERT INTO roomserver_event_search (rowid, body, room_id, event_id, search_key, sender, event_type)
SELECT stream_ordering, json_extract(event_json, '$.content.name'), room_id, event_id,
'content.name', sender, event_type
FROM roomserver_events
WHERE event_type = 'm.room.name' AND NOT outlier AND redacted_by = ''
AND json_type(event_json, '$.content.name') = 'text';
INSERT INTO roomserver_event_search (rowid, body, room_id, event_id, search_key, sender, event_type)
SELECT stream_ordering, json_extract(event_json, '$.content.topic'), room_id, event_id,
'content.topic', sender, event_type
FROM roomserver_events
WHERE event_type = 'm.room.topic' AND NOT outlier AND redacted_by = ''
AND json_type(event_json, '$.content.topic') = 'text';
@@ -0,0 +1,21 @@
-- Phase 10: notifications counted per thread.
--
-- thread_id is the thread an event that notified a user belongs to: the
-- root its m.relates_to names when rel_type is m.thread, and '' for an
-- event on the main timeline. A read receipt for a thread clears that
-- thread's rows and a receipt for the main timeline clears the '' rows,
-- which is what lets reading one thread leave the rest of the room unread.
ALTER TABLE push_notifications ADD COLUMN thread_id TEXT NOT NULL DEFAULT '';
-- The notifications already stored, from the relation 0012 projected onto
-- their events. A notification whose event has gone stays on the main
-- timeline, where an unthreaded or main receipt still clears it.
UPDATE push_notifications SET thread_id = COALESCE((
SELECT e.relates_to_id FROM roomserver_events e
WHERE e.event_id = push_notifications.event_id AND e.rel_type = 'm.thread'
), '');
-- A threaded receipt marks one thread's rows read; the per-thread counts a
-- sync asks for group a room's unread rows by thread.
CREATE INDEX IF NOT EXISTS push_notifications_thread_idx
ON push_notifications (user_id, room_id, thread_id, is_read, event_pos);
+51 -12
View File
@@ -49,6 +49,56 @@ var settingsQueries = sqlstore.SettingsQueries{
Delete: `DELETE FROM server_settings WHERE setting_key = ?`,
}
// searchDialect is the full-text index on FTS5. The table cannot be
// aliased — MATCH and bm25() take its own name — and its rowid is the
// event's stream ordering. bm25() scores better matches lower, so the rank
// is its negation.
var searchDialect = &sqlstore.SearchDialect{
Insert: `INSERT INTO roomserver_event_search (rowid, room_id, event_id, search_key, sender, event_type, body)
VALUES (?, ?, ?, ?, ?, ?, ?)`,
DeleteEvent: `DELETE FROM roomserver_event_search WHERE rowid IN (
SELECT stream_ordering FROM roomserver_events WHERE room_id = ? AND event_id = ?)`,
DeleteRoom: `DELETE FROM roomserver_event_search WHERE room_id = ?`,
From: `roomserver_event_search`,
Name: `roomserver_event_search`,
Position: `roomserver_event_search.rowid`,
Match: `roomserver_event_search MATCH ?`,
Rank: `-bm25(roomserver_event_search)`,
Query: ftsQuery,
}
// ftsQuery builds an FTS5 query from a search term: every word, quoted so
// that nothing in it is read as an operator, and matched as a prefix so
// that a user who has typed "hel" already finds "hello". Words side by
// side are ANDed.
func ftsQuery(term string) string {
words := storage.SearchWords(term)
for i, w := range words {
words[i] = `"` + strings.ReplaceAll(w, `"`, `""`) + `"*`
}
return strings.Join(words, " ")
}
// StoreOptions are the sqlstore options for a SQLite database already
// opened on db: the migrations, statements and error recognition every
// SQLite-backed driver shares. The PocketBase driver runs on a SQLite file
// of its own and uses them as they are.
func StoreOptions(db *sql.DB, log *slog.Logger) sqlstore.Options {
return sqlstore.Options{
DriverName: storage.DriverSQLite,
DB: db,
Dialect: dialect,
MigrationsFS: migrationFiles,
MigrationsDir: "migrations",
Settings: settingsQueries,
// SQLite already uses the `?` placeholders the shared component
// stores are written with, so there is nothing to rebind.
IsUniqueViolation: isUniqueViolation,
Search: searchDialect,
Logger: log,
}
}
// Database is the SQLite implementation of storage.Database.
type Database struct {
*sqlstore.Store
@@ -86,18 +136,7 @@ func Open(ctx context.Context, cfg storage.Config, log *slog.Logger) (storage.Da
return nil, fmt.Errorf("sqlite: connect: %w", err)
}
store, err := sqlstore.New(sqlstore.Options{
DriverName: storage.DriverSQLite,
DB: db,
Dialect: dialect,
MigrationsFS: migrationFiles,
MigrationsDir: "migrations",
Settings: settingsQueries,
// SQLite already uses the `?` placeholders the shared component
// stores are written with, so there is nothing to rebind.
IsUniqueViolation: isUniqueViolation,
Logger: log,
})
store, err := sqlstore.New(StoreOptions(db, log))
if err != nil {
_ = db.Close()
return nil, err
+64 -8
View File
@@ -143,8 +143,8 @@ func (s *Push) DeletePushersForDevices(ctx context.Context, localpart string, de
// to NUMERIC, which does not scan into an int64.
const (
insertNotificationSQL = `INSERT INTO push_notifications
(id, user_id, room_id, event_id, event_pos, ts_ms, highlight, actions, is_read)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
(id, user_id, room_id, event_id, event_pos, thread_id, ts_ms, highlight, actions, is_read)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
bumpRoomCountsSQL = `INSERT INTO push_room_counts
(user_id, room_id, notification_count, highlight_count, stream_pos)
@@ -157,6 +157,10 @@ const (
markReadSQL = `UPDATE push_notifications SET is_read = ?
WHERE user_id = ? AND room_id = ? AND is_read = ? AND event_pos <= ?`
// markThreadReadSQL is markReadSQL for one thread, or for the main
// timeline as thread ''.
markThreadReadSQL = markReadSQL + ` AND thread_id = ?`
countUnreadSQL = `SELECT COUNT(*), CAST(COALESCE(SUM(CASE WHEN highlight THEN 1 ELSE 0 END), 0) AS BIGINT)
FROM push_notifications WHERE user_id = ? AND room_id = ? AND is_read = ?`
@@ -174,7 +178,7 @@ const (
selectUnreadTotalSQL = `SELECT CAST(COALESCE(SUM(notification_count), 0) AS BIGINT)
FROM push_room_counts WHERE user_id = ?`
notificationColumns = `id, user_id, room_id, event_id, event_pos, ts_ms, highlight, actions, is_read`
notificationColumns = `id, user_id, room_id, event_id, event_pos, thread_id, ts_ms, highlight, actions, is_read`
pruneNotificationsSQL = `DELETE FROM push_notifications WHERE is_read = ? AND ts_ms < ?`
)
@@ -196,7 +200,7 @@ func (s *Push) AddNotifications(ctx context.Context, ns []storage.NewNotificatio
return err
}
if _, err := tx.ExecContext(ctx, s.q(insertNotificationSQL), pos, n.UserID, n.RoomID, n.EventID,
n.EventPos, n.TimestampMS, n.Highlight, string(n.Actions), false); err != nil {
n.EventPos, n.ThreadID, n.TimestampMS, n.Highlight, string(n.Actions), false); err != nil {
return fmt.Errorf("write a notification for %q: %w", n.UserID, err)
}
highlight := 0
@@ -218,14 +222,19 @@ func (s *Push) AddNotifications(ctx context.Context, ns []storage.NewNotificatio
// The counts are recounted from the rows that are still unread rather than
// decremented, so that whatever got them out of step — a crash between
// two writes of an older version, a row pruned by hand — is put right by
// the next receipt.
func (s *Push) MarkNotificationsRead(ctx context.Context, userID, roomID string, upTo int64) (int64, bool, error) {
// the next receipt. The room's counts stay the room's total across every
// thread whichever thread the receipt read.
func (s *Push) MarkNotificationsRead(ctx context.Context, userID, roomID, thread string, upTo int64) (int64, bool, error) {
var (
pos int64
moved bool
)
err := s.tx(ctx, func(tx *sql.Tx) error {
res, err := tx.ExecContext(ctx, s.q(markReadSQL), true, userID, roomID, false, upTo)
query, args := markReadSQL, []any{true, userID, roomID, false, upTo}
if thread != storage.AllThreads {
query, args = markThreadReadSQL, append(args, thread)
}
res, err := tx.ExecContext(ctx, s.q(query), args...)
if err != nil {
return fmt.Errorf("mark %q's notifications read: %w", userID, err)
}
@@ -291,6 +300,53 @@ func (s *Push) NotificationCounts(ctx context.Context, userID string, roomIDs []
return out, nil
}
// ThreadNotificationCounts implements storage.PushStore.
//
// These are counted from the unread rows rather than kept like the room
// totals: a room has few unread notifications at a time, they are a range
// of an index, and a table of per-thread counts would be one more thing
// every write and every receipt has to keep in step.
func (s *Push) ThreadNotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]map[string]storage.NotificationCounts, error) {
out := make(map[string]map[string]storage.NotificationCounts, len(roomIDs))
if len(roomIDs) == 0 {
return out, nil
}
query := `SELECT room_id, thread_id, COUNT(*),
CAST(COALESCE(SUM(CASE WHEN highlight THEN 1 ELSE 0 END), 0) AS BIGINT)
FROM push_notifications
WHERE user_id = ? AND is_read = ? AND room_id IN (` + placeholders(len(roomIDs)) + `)
GROUP BY room_id, thread_id`
args := make([]any, 0, len(roomIDs)+2)
args = append(args, userID, false)
for _, id := range roomIDs {
args = append(args, id)
}
rows, err := s.db.QueryContext(ctx, s.q(query), args...)
if err != nil {
return nil, fmt.Errorf("read %q's thread notification counts: %w", userID, err)
}
defer func() { _ = rows.Close() }()
for rows.Next() {
var roomID, thread string
var notifications, highlights int64
if err := rows.Scan(&roomID, &thread, &notifications, &highlights); err != nil {
return nil, fmt.Errorf("read %q's thread notification counts: %w", userID, err)
}
if out[roomID] == nil {
out[roomID] = map[string]storage.NotificationCounts{}
}
out[roomID][thread] = storage.NotificationCounts{
Notifications: int(notifications),
Highlights: int(highlights),
}
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("read %q's thread notification counts: %w", userID, err)
}
return out, nil
}
// NotificationCountChanges implements storage.PushStore.
func (s *Push) NotificationCountChanges(ctx context.Context, userID string, from, to int64) ([]string, error) {
rows, err := s.db.QueryContext(ctx, s.q(selectCountChangesSQL), userID, from, to)
@@ -345,7 +401,7 @@ func (s *Push) Notifications(ctx context.Context, userID string, before int64, l
for rows.Next() {
var n storage.Notification
var actions string
if err := rows.Scan(&n.ID, &n.UserID, &n.RoomID, &n.EventID, &n.EventPos, &n.TimestampMS,
if err := rows.Scan(&n.ID, &n.UserID, &n.RoomID, &n.EventID, &n.EventPos, &n.ThreadID, &n.TimestampMS,
&n.Highlight, &actions, &n.Read); err != nil {
return nil, fmt.Errorf("read %q's notifications: %w", userID, err)
}
@@ -0,0 +1,495 @@
package sqlstore
import (
"context"
"database/sql"
"fmt"
"strings"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// The relation and search half of the room store: the reads behind
// /relations, /threads, the bundled summaries in unsigned.m.relations, and
// /search.
//
// Relations need nothing but the two columns on roomserver_events and their
// index, and are written once for both engines like everything else here.
// Search is the one place the engines really differ — SQLite has FTS5,
// PostgreSQL has tsvector — so the statements that touch the index come
// from the driver as a SearchDialect, and only the shape of the query is
// shared.
// SearchDialect is one engine's full-text index.
//
// Every statement is written against the table roomserver_event_search,
// referred to as Name where it is read, with `?` placeholders that are
// rebound like any other statement.
type SearchDialect struct {
// Insert adds one event, taking (stream_ordering, room_id, event_id,
// search_key, sender, event_type, body) in that order.
Insert string
// DeleteEvent removes an event's row, taking (room_id, event_id).
DeleteEvent string
// DeleteRoom removes a room's rows, taking (room_id).
DeleteRoom string
// From is the FROM clause the search reads, and Name what its columns
// are qualified with. SQLite's FTS5 cannot be aliased — MATCH and
// bm25() take the table's own name — so the two are separate.
From string
Name string
// Position is the column holding a row's stream ordering.
Position string
// Match is the condition selecting the rows that match. It takes the
// engine query once.
Match string
// Rank is the relevance of a matching row, higher being better. It
// takes the engine query RankArgs times.
Rank string
RankArgs int
// Query turns what a user typed into the engine's query syntax. It
// returns "" for a term with nothing searchable in it, which matches
// nothing.
Query func(term string) string
}
// WithSearch sets the full-text index the store writes and reads.
func (s *Rooms) WithSearch(d *SearchDialect) *Rooms {
s.search = d
return s
}
// indexEvent adds an event that has just joined a timeline to the search
// index, if it has anything to find it by.
func (s *Rooms) indexEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEvent, ordering int64) error {
if s.search == nil || e.SearchKey == "" || strings.TrimSpace(e.SearchText) == "" {
return nil
}
_, err := tx.ExecContext(ctx, s.q(s.search.Insert),
ordering, e.RoomID, e.EventID, e.SearchKey, e.Sender, e.Type, e.SearchText)
if err != nil {
return fmt.Errorf("index event %s for search: %w", e.EventID, err)
}
return nil
}
// unindexEvent removes an event from the search index.
func (s *Rooms) unindexEvent(ctx context.Context, tx *sql.Tx, roomID, eventID string) error {
if s.search == nil {
return nil
}
if _, err := tx.ExecContext(ctx, s.q(s.search.DeleteEvent), roomID, eventID); err != nil {
return fmt.Errorf("remove event %s from the search index: %w", eventID, err)
}
return nil
}
// Relations implements storage.RoomStore.
func (s *Rooms) Relations(ctx context.Context, req storage.RelationsRequest) ([]storage.StoredEvent, error) {
if req.Limit <= 0 || len(req.ParentIDs) == 0 {
return nil, nil
}
conditions := []string{
`e.room_id = ?`,
`e.relates_to_id IN (` + placeholders(len(req.ParentIDs)) + `)`,
`NOT e.outlier`,
}
args := []any{req.RoomID}
for _, id := range req.ParentIDs {
args = append(args, id)
}
if req.RelType != "" {
conditions = append(conditions, `e.rel_type = ?`)
args = append(args, req.RelType)
} else {
// A parent's own row has relates_to_id '' and can never match an
// IN list of event IDs, but '' itself could be passed; it names no
// event and has no children.
conditions = append(conditions, `e.rel_type <> ''`)
}
if req.EventType != "" {
conditions = append(conditions, `e.event_type = ?`)
args = append(args, req.EventType)
}
if req.Low != 0 {
conditions = append(conditions, `e.stream_ordering > ?`)
args = append(args, req.Low)
}
if req.High != 0 {
conditions = append(conditions, `e.stream_ordering <= ?`)
args = append(args, req.High)
}
order := `ASC`
if req.Direction == storage.DirectionBackward {
order = `DESC`
}
args = append(args, req.Limit)
query := `SELECT ` + eventColumns + `
FROM roomserver_events e
WHERE ` + strings.Join(conditions, " AND ") + `
ORDER BY e.stream_ordering ` + order + `
LIMIT ?`
rows, err := s.db.QueryContext(ctx, s.q(query), args...)
if err != nil {
return nil, fmt.Errorf("read the relations of %v in %q: %w", req.ParentIDs, req.RoomID, err)
}
events, err := scanEvents(rows)
if err != nil {
return nil, fmt.Errorf("read the relations of %v in %q: %w", req.ParentIDs, req.RoomID, err)
}
return events, nil
}
// ChildEventIDs implements storage.RoomStore.
func (s *Rooms) ChildEventIDs(ctx context.Context, roomID string, parentIDs []string, relType, eventType string) ([]string, error) {
if len(parentIDs) == 0 {
return nil, nil
}
query := `SELECT event_id FROM roomserver_events
WHERE room_id = ? AND relates_to_id IN (` + placeholders(len(parentIDs)) + `)
AND rel_type <> '' AND NOT outlier`
args := []any{roomID}
for _, id := range parentIDs {
args = append(args, id)
}
if relType != "" {
query += ` AND rel_type = ?`
args = append(args, relType)
}
if eventType != "" {
query += ` AND event_type = ?`
args = append(args, eventType)
}
query += ` ORDER BY stream_ordering`
ids, err := s.queryStrings(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("read the children of %v in %q: %w", parentIDs, roomID, err)
}
return ids, nil
}
// RelationAggregates implements storage.RoomStore.
//
// It groups by parent, relation type and sender, and joins each group's
// newest position back onto the events table for its event ID: a portable
// spelling of "the row with the largest stream ordering in each group",
// which neither engine can say in one aggregate.
func (s *Rooms) RelationAggregates(ctx context.Context, roomID string, parentIDs []string, high int64) ([]storage.RelationAggregate, error) {
if len(parentIDs) == 0 {
return nil, nil
}
inner := `SELECT relates_to_id, rel_type, sender, COUNT(*) AS n, MAX(stream_ordering) AS latest
FROM roomserver_events
WHERE room_id = ? AND relates_to_id IN (` + placeholders(len(parentIDs)) + `)
AND rel_type IN (?, ?, ?) AND NOT outlier`
args := []any{roomID}
for _, id := range parentIDs {
args = append(args, id)
}
args = append(args, storage.RelThread, storage.RelReplace, storage.RelReference)
if high != 0 {
inner += ` AND stream_ordering <= ?`
args = append(args, high)
}
inner += ` GROUP BY relates_to_id, rel_type, sender`
query := `SELECT g.relates_to_id, g.rel_type, g.sender, g.n, g.latest, e.event_id
FROM (` + inner + `) g
JOIN roomserver_events e ON e.stream_ordering = g.latest`
rows, err := s.db.QueryContext(ctx, s.q(query), args...)
if err != nil {
return nil, fmt.Errorf("summarise the relations of %d events in %q: %w", len(parentIDs), roomID, err)
}
defer func() { _ = rows.Close() }()
var out []storage.RelationAggregate
for rows.Next() {
var a storage.RelationAggregate
if err := rows.Scan(&a.ParentID, &a.RelType, &a.Sender, &a.Count, &a.LatestOrdering, &a.LatestEventID); err != nil {
return nil, fmt.Errorf("summarise the relations in %q: %w", roomID, err)
}
out = append(out, a)
}
return out, rows.Err()
}
// RelationEventIDs implements storage.RoomStore.
//
// The limit is per parent in spirit and applied over the whole set: the
// summary it feeds is a sample, and a parent with more children than the
// limit is rare enough that one query beats one per parent.
func (s *Rooms) RelationEventIDs(ctx context.Context, roomID string, parentIDs []string, relType string, high int64, limit int) (map[string][]string, error) {
if len(parentIDs) == 0 || limit <= 0 {
return nil, nil
}
query := `SELECT relates_to_id, event_id FROM roomserver_events
WHERE room_id = ? AND relates_to_id IN (` + placeholders(len(parentIDs)) + `)
AND rel_type = ? AND NOT outlier`
args := []any{roomID}
for _, id := range parentIDs {
args = append(args, id)
}
args = append(args, relType)
if high != 0 {
query += ` AND stream_ordering <= ?`
args = append(args, high)
}
query += ` ORDER BY stream_ordering LIMIT ?`
args = append(args, limit*len(parentIDs))
rows, err := s.db.QueryContext(ctx, s.q(query), args...)
if err != nil {
return nil, fmt.Errorf("read the %s relations in %q: %w", relType, roomID, err)
}
defer func() { _ = rows.Close() }()
out := map[string][]string{}
for rows.Next() {
var parent, id string
if err := rows.Scan(&parent, &id); err != nil {
return nil, fmt.Errorf("read the %s relations in %q: %w", relType, roomID, err)
}
if len(out[parent]) < limit {
out[parent] = append(out[parent], id)
}
}
return out, rows.Err()
}
// Threads implements storage.RoomStore.
//
// A thread is a root with at least one m.thread child, and is ordered by its
// newest child. The root has to be an event of the room this server holds
// in its timeline: a thread whose root is missing — not yet backfilled, or
// never sent here — cannot be shown as a thread at all.
func (s *Rooms) Threads(ctx context.Context, req storage.ThreadsRequest) ([]storage.Thread, error) {
if req.Limit <= 0 {
return nil, nil
}
inner := `SELECT relates_to_id AS root, MAX(stream_ordering) AS latest
FROM roomserver_events
WHERE room_id = ? AND rel_type = ? AND NOT outlier`
args := []any{req.RoomID, storage.RelThread}
if req.High != 0 {
inner += ` AND stream_ordering <= ?`
args = append(args, req.High)
}
inner += ` GROUP BY relates_to_id`
conditions := []string{`r.room_id = ?`, `NOT r.outlier`}
args = append(args, req.RoomID)
if req.Before != 0 {
conditions = append(conditions, `g.latest < ?`)
args = append(args, req.Before)
}
if req.ParticipatedBy != "" {
conditions = append(conditions, `(r.sender = ? OR EXISTS (
SELECT 1 FROM roomserver_events p
WHERE p.room_id = r.room_id AND p.relates_to_id = g.root AND p.rel_type = ? AND p.sender = ?))`)
args = append(args, req.ParticipatedBy, storage.RelThread, req.ParticipatedBy)
}
args = append(args, req.Limit)
query := `SELECT g.root, g.latest
FROM (` + inner + `) g
JOIN roomserver_events r ON r.event_id = g.root
WHERE ` + strings.Join(conditions, " AND ") + `
ORDER BY g.latest DESC
LIMIT ?`
rows, err := s.db.QueryContext(ctx, s.q(query), args...)
if err != nil {
return nil, fmt.Errorf("read the threads of %q: %w", req.RoomID, err)
}
defer func() { _ = rows.Close() }()
var out []storage.Thread
for rows.Next() {
var t storage.Thread
if err := rows.Scan(&t.RootID, &t.LatestOrdering); err != nil {
return nil, fmt.Errorf("read the threads of %q: %w", req.RoomID, err)
}
out = append(out, t)
}
return out, rows.Err()
}
// searchKeys are the fields the index holds.
var searchKeys = []string{"content.body", "content.name", "content.topic"}
// Search implements storage.RoomStore.
func (s *Rooms) Search(ctx context.Context, req storage.SearchRequest) ([]storage.SearchResult, int, error) {
if s.search == nil || req.Limit <= 0 || len(req.Rooms) == 0 {
return nil, 0, nil
}
term := s.search.Query(req.Term)
if term == "" {
return nil, 0, nil
}
pos := s.search.Position
col := func(name string) string { return s.search.Name + "." + name }
conditions := []string{s.search.Match}
args := []any{term}
// One bounded clause per room: what a user may search of a room is a
// window of its stream, and the windows differ from room to room.
rooms := make([]string, 0, len(req.Rooms))
for _, r := range req.Rooms {
clause := col(`room_id`) + ` = ?`
args = append(args, r.RoomID)
if r.Low != 0 {
clause += ` AND ` + pos + ` > ?`
args = append(args, r.Low)
}
if r.High != 0 {
clause += ` AND ` + pos + ` <= ?`
args = append(args, r.High)
}
rooms = append(rooms, `(`+clause+`)`)
}
conditions = append(conditions, `(`+strings.Join(rooms, ` OR `)+`)`)
keys := req.Keys
if len(keys) == 0 {
keys = searchKeys
}
conditions = append(conditions, col(`search_key`)+` IN (`+placeholders(len(keys))+`)`)
for _, k := range keys {
args = append(args, k)
}
if clause, filterArgs := filterConditionsOn(col(`event_type`), col(`sender`), req.Filter); clause != "" {
conditions = append(conditions, clause)
args = append(args, filterArgs...)
}
where := strings.Join(conditions, " AND ")
var total int
countQuery := `SELECT COUNT(*) FROM ` + s.search.From + ` WHERE ` + where
if err := s.db.QueryRowContext(ctx, s.q(countQuery), args...).Scan(&total); err != nil {
return nil, 0, fmt.Errorf("count search results: %w", err)
}
pageWhere := where
pageArgs := append([]any{}, args...)
if req.Order == storage.SearchByRecent && req.Before != 0 {
pageWhere += ` AND ` + pos + ` < ?`
pageArgs = append(pageArgs, req.Before)
}
rankArgs := make([]any, 0, s.search.RankArgs)
for range s.search.RankArgs {
rankArgs = append(rankArgs, term)
}
order := `rank DESC, ` + pos + ` DESC`
if req.Order == storage.SearchByRecent {
order = pos + ` DESC`
}
query := `SELECT ` + pos + `, ` + s.search.Rank + ` AS rank
FROM ` + s.search.From + `
WHERE ` + pageWhere + `
ORDER BY ` + order + `
LIMIT ? OFFSET ?`
offset := req.Offset
if req.Order == storage.SearchByRecent {
offset = 0
}
queryArgs := append(rankArgs, pageArgs...)
queryArgs = append(queryArgs, req.Limit, offset)
hits, err := s.searchHits(ctx, query, queryArgs)
if err != nil {
return nil, 0, err
}
if len(hits) == 0 {
return nil, total, nil
}
// The events themselves, read by position in one query and put back in
// the order the search ranked them.
eventArgs := make([]any, 0, len(hits))
for _, h := range hits {
eventArgs = append(eventArgs, h.pos)
}
eventRows, err := s.db.QueryContext(ctx, s.q(`SELECT `+eventColumns+`
FROM roomserver_events e WHERE e.stream_ordering IN (`+placeholders(len(hits))+`)`), eventArgs...)
if err != nil {
return nil, 0, fmt.Errorf("read search results: %w", err)
}
events, err := scanEvents(eventRows)
if err != nil {
return nil, 0, fmt.Errorf("read search results: %w", err)
}
byPos := make(map[int64]storage.StoredEvent, len(events))
for _, e := range events {
byPos[e.StreamOrdering] = e
}
out := make([]storage.SearchResult, 0, len(hits))
for _, h := range hits {
// An index row whose event has gone — purged under a search in
// flight — is skipped rather than reported.
if e, ok := byPos[h.pos]; ok {
out = append(out, storage.SearchResult{Event: e, Rank: h.rank})
}
}
return out, total, nil
}
// searchHit is one row of a search page: where the event is, and how well
// it matched.
type searchHit struct {
pos int64
rank float64
}
// searchHits runs the page query of a search.
func (s *Rooms) searchHits(ctx context.Context, query string, args []any) ([]searchHit, error) {
rows, err := s.db.QueryContext(ctx, s.q(query), args...)
if err != nil {
return nil, fmt.Errorf("search: %w", err)
}
defer func() { _ = rows.Close() }()
var hits []searchHit
for rows.Next() {
var h searchHit
if err := rows.Scan(&h.pos, &h.rank); err != nil {
return nil, fmt.Errorf("search: %w", err)
}
hits = append(hits, h)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("search: %w", err)
}
return hits, nil
}
// queryStrings runs a query returning one string column.
func (s *Rooms) queryStrings(ctx context.Context, query string, args ...any) ([]string, error) {
rows, err := s.db.QueryContext(ctx, s.q(query), args...)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
var out []string
for rows.Next() {
var v string
if err := rows.Scan(&v); err != nil {
return nil, err
}
out = append(out, v)
}
return out, rows.Err()
}
+56 -13
View File
@@ -25,6 +25,9 @@ type Rooms struct {
db *sql.DB
rebind func(string) string
isUnique func(error) bool
// search is the engine's full-text index, nil for none.
search *SearchDialect
}
// NewRooms builds a room store.
@@ -266,9 +269,10 @@ const (
insertEventSQL = `
INSERT INTO roomserver_events (
event_id, room_id, stream_ordering, event_type, state_key,
sender, depth, origin_server_ts, event_json, state_before_id, outlier)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
selectOutlierSQL = `SELECT outlier FROM roomserver_events WHERE event_id = ?`
sender, depth, origin_server_ts, event_json, state_before_id, outlier,
relates_to_id, rel_type)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
selectOutlierSQL = `SELECT outlier, redacted_by FROM roomserver_events WHERE event_id = ?`
promoteOutlierSQL = `
UPDATE roomserver_events
SET stream_ordering = ?, state_before_id = ?, outlier = ?
@@ -284,8 +288,11 @@ const (
// the event ID says the two copies are the same event, and the stored one
// may already have been redacted.
func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEvent, ordering, snapshotID int64, outlier bool) error {
var existingOutlier bool
err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), e.EventID).Scan(&existingOutlier)
var (
existingOutlier bool
redactedBy string
)
err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), e.EventID).Scan(&existingOutlier, &redactedBy)
switch {
case err == nil:
if existingOutlier && !outlier {
@@ -293,6 +300,13 @@ func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEve
ordering, snapshotID, false, e.EventID, true); err != nil {
return fmt.Errorf("promote outlier: %w", err)
}
// It joins the timeline, so it joins the search index — unless
// the copy already held was redacted, whose text is gone.
if redactedBy == "" {
if err := s.indexEvent(ctx, tx, e, ordering); err != nil {
return err
}
}
}
// The same event twice is not a failure worth propagating: an
// event ID is a hash of its contents, so the second copy says
@@ -304,13 +318,17 @@ func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEve
_, err = tx.ExecContext(ctx, s.q(insertEventSQL),
e.EventID, e.RoomID, ordering, e.Type, e.StateKey,
e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, outlier)
e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, outlier,
e.RelatesTo, e.RelType)
if err != nil {
if s.isUnique(err) {
return nil
}
return fmt.Errorf("insert event: %w", err)
}
if !outlier {
return s.indexEvent(ctx, tx, e, ordering)
}
return nil
}
@@ -403,9 +421,13 @@ func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []stora
// which is already in the timeline and left alone.
var write []storage.BackfillEvent
promote := map[string]bool{}
redacted := map[string]bool{}
for _, b := range events {
var outlier bool
err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), b.Event.EventID).Scan(&outlier)
var (
outlier bool
redactedBy string
)
err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), b.Event.EventID).Scan(&outlier, &redactedBy)
switch {
case errors.Is(err, sql.ErrNoRows):
write = append(write, b)
@@ -414,6 +436,7 @@ func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []stora
case outlier:
write = append(write, b)
promote[b.Event.EventID] = true
redacted[b.Event.EventID] = redactedBy != ""
}
}
if len(write) == 0 {
@@ -439,13 +462,22 @@ func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []stora
ordering, snapshotID, false, e.EventID, true); err != nil {
return fmt.Errorf("promote outlier %s: %w", e.EventID, err)
}
if !redacted[e.EventID] {
if err := s.indexEvent(ctx, tx, e, ordering); err != nil {
return err
}
}
continue
}
if _, err := tx.ExecContext(ctx, s.q(insertEventSQL),
e.EventID, e.RoomID, ordering, e.Type, e.StateKey,
e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, false); err != nil {
e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, false,
e.RelatesTo, e.RelType); err != nil {
return fmt.Errorf("insert event %s: %w", e.EventID, err)
}
if err := s.indexEvent(ctx, tx, e, ordering); err != nil {
return err
}
}
return nil
})
@@ -456,7 +488,7 @@ func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []stora
}
const applyRedactionSQL = `
UPDATE roomserver_events SET event_json = ?, redacted_by = ?
UPDATE roomserver_events SET event_json = ?, redacted_by = ?, relates_to_id = '', rel_type = ''
WHERE room_id = ? AND event_id = ? AND redacted_by = ''`
// applyRedaction replaces a stored event with its redacted form.
@@ -470,7 +502,9 @@ func (s *Rooms) applyRedaction(ctx context.Context, tx *sql.Tx, roomID, targetID
if err != nil {
return fmt.Errorf("apply redaction to %s: %w", targetID, err)
}
return nil
// What a redacted event said is gone, and so is its way of being found
// by what it said.
return s.unindexEvent(ctx, tx, roomID, targetID)
}
const (
@@ -574,7 +608,8 @@ func (s *Rooms) recordTransaction(ctx context.Context, tx *sql.Tx, ref storage.T
// scanEvent expects.
const eventColumns = `
e.room_id, e.event_id, e.event_type, e.state_key, e.sender,
e.depth, e.origin_server_ts, e.event_json, e.stream_ordering, e.redacted_by, e.outlier`
e.depth, e.origin_server_ts, e.event_json, e.stream_ordering, e.redacted_by, e.outlier,
e.relates_to_id, e.rel_type`
// scanEvent reads one row of eventColumns.
func scanEvent(scan func(...any) error) (storage.StoredEvent, error) {
@@ -584,7 +619,8 @@ func scanEvent(scan func(...any) error) (storage.StoredEvent, error) {
body string
)
err := scan(&e.RoomID, &e.EventID, &e.Type, &stateKey, &e.Sender,
&e.Depth, &e.OriginServerTS, &body, &e.StreamOrdering, &e.RedactedBy, &e.Outlier)
&e.Depth, &e.OriginServerTS, &body, &e.StreamOrdering, &e.RedactedBy, &e.Outlier,
&e.RelatesTo, &e.RelType)
if err != nil {
return storage.StoredEvent{}, err
}
@@ -1087,6 +1123,13 @@ var purgeStatements = []string{
// PurgeRoom implements storage.RoomStore.
func (s *Rooms) PurgeRoom(ctx context.Context, roomID string) error {
err := s.tx(ctx, func(tx *sql.Tx) error {
// The search index has no foreign key to cascade from: SQLite's is
// a virtual table.
if s.search != nil {
if _, err := tx.ExecContext(ctx, s.q(s.search.DeleteRoom), roomID); err != nil {
return err
}
}
for i, stmt := range purgeStatements {
result, err := tx.ExecContext(ctx, s.q(stmt), roomID)
if err != nil {
+5 -1
View File
@@ -45,6 +45,10 @@ type Options struct {
// without a check-then-insert race.
IsUniqueViolation func(error) bool
// Search is this engine's full-text search. Nil leaves the index
// unwritten and /search finding nothing.
Search *SearchDialect
// Logger is used for migration progress.
Logger *slog.Logger
}
@@ -81,7 +85,7 @@ func New(opts Options) (*Store, error) {
log: opts.Logger,
settings: NewSettings(opts.DB, opts.Settings),
users: NewUsers(opts.DB, opts.Rebind, opts.IsUniqueViolation),
rooms: NewRooms(opts.DB, opts.Rebind, opts.IsUniqueViolation),
rooms: NewRooms(opts.DB, opts.Rebind, opts.IsUniqueViolation).WithSearch(opts.Search),
sync: NewSync(opts.DB, opts.Rebind, opts.IsUniqueViolation),
fed: NewFederation(opts.DB, opts.Rebind),
media: NewMedia(opts.DB, opts.Rebind, opts.IsUniqueViolation),
+52 -4
View File
@@ -183,6 +183,12 @@ func (s *Rooms) StreamEvents(ctx context.Context, after, upTo int64, limit int)
// not_types win over types for an event matching both — the behaviour the
// spec requires.
func filterConditions(f storage.TimelineFilter) (string, []any) {
return filterConditionsOn(`e.event_type`, `e.sender`, f)
}
// filterConditionsOn is filterConditions against named columns, for a
// table other than roomserver_events.
func filterConditionsOn(typeColumn, senderColumn string, f storage.TimelineFilter) (string, []any) {
var (
clauses []string
args []any
@@ -198,10 +204,10 @@ func filterConditions(f storage.TimelineFilter) (string, []any) {
clauses = append(clauses, clause)
args = append(args, clauseArgs...)
}
add(`e.event_type`, f.Types, false)
add(`e.event_type`, f.NotTypes, true)
add(`e.sender`, f.Senders, false)
add(`e.sender`, f.NotSenders, true)
add(typeColumn, f.Types, false)
add(typeColumn, f.NotTypes, true)
add(senderColumn, f.Senders, false)
add(senderColumn, f.NotSenders, true)
if len(clauses) == 0 {
return "", nil
@@ -299,6 +305,48 @@ func (s *Rooms) RoomsWithEvents(ctx context.Context, roomIDs []string, from, to
return out, rows.Err()
}
// LatestEventPositions implements storage.RoomStore.
func (s *Rooms) LatestEventPositions(ctx context.Context, roomIDs, types []string, upTo int64) (map[string]int64, error) {
if len(roomIDs) == 0 || len(types) == 0 {
return map[string]int64{}, nil
}
query := `
SELECT room_id, MAX(stream_ordering) FROM roomserver_events
WHERE room_id IN (` + placeholders(len(roomIDs)) + `)
AND event_type IN (` + placeholders(len(types)) + `) AND NOT outlier`
args := make([]any, 0, len(roomIDs)+len(types)+1)
for _, id := range roomIDs {
args = append(args, id)
}
for _, t := range types {
args = append(args, t)
}
if upTo != 0 {
query += ` AND stream_ordering <= ?`
args = append(args, upTo)
}
query += ` GROUP BY room_id`
rows, err := s.db.QueryContext(ctx, s.q(query), args...)
if err != nil {
return nil, fmt.Errorf("find the latest activity of %d rooms: %w", len(roomIDs), err)
}
defer func() { _ = rows.Close() }()
out := make(map[string]int64, len(roomIDs))
for rows.Next() {
var (
roomID string
pos int64
)
if err := rows.Scan(&roomID, &pos); err != nil {
return nil, fmt.Errorf("find the latest activity of %d rooms: %w", len(roomIDs), err)
}
out[roomID] = pos
}
return out, rows.Err()
}
const selectEventAtStreamSQL = `
SELECT e.event_id, e.event_type, e.state_key, e.state_before_id
FROM roomserver_events e
+15
View File
@@ -40,6 +40,21 @@ type Config struct {
MaxOpenConns int
MaxIdleConns int
ConnMaxLifetime time.Duration
// S3 is the bucket a driver with its own file storage puts media in.
// Only the pocketbase driver reads it.
S3 S3Config
}
// S3Config is an S3-compatible bucket.
type S3Config struct {
Enabled bool
Bucket string
Region string
Endpoint string
AccessKey string
Secret string
ForcePathStyle bool
}
// Database is the handle every component gets. Component-specific stores
+57 -9
View File
@@ -2,9 +2,11 @@ package storage_test
import (
"errors"
"io"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"github.com/dariuszjaniak/warp/server/internal/storage"
@@ -28,6 +30,8 @@ func openTestDB(t *testing.T, driver string) storage.Database {
t.Skip("set WARP_TEST_POSTGRES_DSN to test the postgres driver")
}
cfg.ConnectionString = dsn
case storage.DriverPocketBase:
cfg.ConnectionString = filepath.Join(t.TempDir(), "pb_data")
default:
t.Fatalf("no test setup for driver %q", driver)
}
@@ -40,8 +44,9 @@ func openTestDB(t *testing.T, driver string) storage.Database {
return db
}
// testDrivers are run by every storage test.
var testDrivers = []string{storage.DriverSQLite, storage.DriverPostgres}
// testDrivers are run by every storage test. PocketBase is here because
// parity with sqlite is its whole contract.
var testDrivers = []string{storage.DriverSQLite, storage.DriverPostgres, storage.DriverPocketBase}
func TestDriversRegistered(t *testing.T) {
got := storage.Drivers()
@@ -59,13 +64,56 @@ func TestOpenUnknownDriver(t *testing.T) {
}
}
func TestPocketBaseIsNotImplemented(t *testing.T) {
_, err := storage.Open(t.Context(), storage.Config{
Driver: storage.DriverPocketBase,
ConnectionString: "./pb_data",
}, nil)
if !errors.Is(err, storage.ErrNotImplemented) {
t.Fatalf("expected ErrNotImplemented, got %v", err)
func TestPocketBaseNeedsADataDirectory(t *testing.T) {
if _, err := storage.Open(t.Context(), storage.Config{Driver: storage.DriverPocketBase}, nil); err == nil {
t.Fatal("expected an error for a pocketbase driver with no data directory")
}
}
func TestPocketBaseKeepsMediaInItsFilesystem(t *testing.T) {
db := openTestDB(t, storage.DriverPocketBase)
provider, ok := db.(storage.BlobProvider)
if !ok {
t.Fatal("the pocketbase driver does not provide its own blob store")
}
blobs := provider.Blobs()
ctx := t.Context()
info, err := blobs.Put(ctx, strings.NewReader("hello media"), 1024)
if err != nil {
t.Fatalf("Put: %v", err)
}
if info.Size != 11 || len(info.SHA256) != 64 {
t.Fatalf("Put returned %+v", info)
}
if again, err := blobs.Put(ctx, strings.NewReader("hello media"), 1024); err != nil || again != info {
t.Errorf("the same content again = %+v, %v", again, err)
}
if _, err := blobs.Put(ctx, strings.NewReader("too long"), 3); !errors.Is(err, storage.ErrBlobTooLarge) {
t.Errorf("an oversize Put = %v, want ErrBlobTooLarge", err)
}
r, err := blobs.Open(ctx, info.SHA256)
if err != nil {
t.Fatalf("Open: %v", err)
}
body, err := io.ReadAll(r)
_ = r.Close()
if err != nil || string(body) != "hello media" {
t.Errorf("read back %q, %v", body, err)
}
if err := blobs.Delete(ctx, info.SHA256); err != nil {
t.Fatalf("Delete: %v", err)
}
if err := blobs.Delete(ctx, info.SHA256); err != nil {
t.Errorf("deleting twice = %v", err)
}
if _, err := blobs.Open(ctx, info.SHA256); !errors.Is(err, storage.ErrNoSuchBlob) {
t.Errorf("Open after Delete = %v, want ErrNoSuchBlob", err)
}
if _, err := blobs.Open(ctx, "../../etc/passwd"); !errors.Is(err, storage.ErrNoSuchBlob) {
t.Errorf("Open of a non-hash = %v", err)
}
}
+10 -2
View File
@@ -98,8 +98,10 @@ type RoomEventFilter struct {
// this were set; see roomserver.shapeState.
IncludeRedundantMembers bool `json:"include_redundant_members,omitempty"`
// UnreadThreadNotifications is a threads feature; threads are
// PLAN.md phase 10.
// UnreadThreadNotifications splits a joined room's unread counts by
// thread: unread_notifications counts the main timeline only, and
// unread_thread_notifications has the counts of each thread. Only the
// room timeline's flag is read, which is where the spec puts it.
UnreadThreadNotifications bool `json:"unread_thread_notifications,omitempty"`
}
@@ -221,6 +223,12 @@ func (f Filter) LazyLoadMembers() bool {
return f.Room != nil && f.Room.Timeline != nil && f.Room.Timeline.LazyLoadMembers
}
// UnreadThreadNotifications reports whether the filter asks for unread
// counts per thread.
func (f Filter) UnreadThreadNotifications() bool {
return f.Room != nil && f.Room.Timeline != nil && f.Room.Timeline.UnreadThreadNotifications
}
// IncludeLeave reports whether the rooms the user has left should be
// reported in full rather than only when the leave is new.
func (f Filter) IncludeLeave() bool {
+40 -1
View File
@@ -3,6 +3,8 @@ package syncapi
import (
"context"
"slices"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// pushRulesType is the account data push rules travel as. The push server
@@ -24,7 +26,15 @@ type UnreadNotifications struct {
// counts moved: a room is in a response because something in it changed,
// and a client showing a new message wants the count beside it whether or
// not this particular sync is the one that moved it.
func (s *API) unreadNotifications(ctx context.Context, userID string, resp *Response) error {
//
// With byThread — the filter's unread_thread_notifications — the counts
// are split: unread_notifications is the main timeline's, and each thread
// with something unread is in unread_thread_notifications. A thread read
// to the end drops out of the map rather than being reported as zero,
// which the spec allows and a client takes the same way: the room is in
// the response because its counts moved, and the map it is sent stands
// for all of the room's threads, so a thread missing from it is read.
func (s *API) unreadNotifications(ctx context.Context, userID string, byThread bool, resp *Response) error {
if s.push == nil || len(resp.Rooms.Join) == 0 {
return nil
}
@@ -34,6 +44,35 @@ func (s *API) unreadNotifications(ctx context.Context, userID string, resp *Resp
}
slices.Sort(roomIDs)
if byThread {
counts, err := s.push.ThreadNotificationCounts(ctx, userID, roomIDs)
if err != nil {
return err
}
for _, roomID := range roomIDs {
room := resp.Rooms.Join[roomID]
main := counts[roomID][storage.ThreadMain]
room.UnreadNotifications = &UnreadNotifications{
NotificationCount: main.Notifications,
HighlightCount: main.Highlights,
}
for thread, c := range counts[roomID] {
if thread == storage.ThreadMain {
continue
}
if room.UnreadThreadNotifications == nil {
room.UnreadThreadNotifications = map[string]UnreadNotifications{}
}
room.UnreadThreadNotifications[thread] = UnreadNotifications{
NotificationCount: c.Notifications,
HighlightCount: c.Highlights,
}
}
resp.Rooms.Join[roomID] = room
}
return nil
}
counts, err := s.push.NotificationCounts(ctx, userID, roomIDs)
if err != nil {
return err
+197
View File
@@ -0,0 +1,197 @@
package syncapi
import (
"context"
"fmt"
"strconv"
"strings"
"github.com/dariuszjaniak/warp/server/internal/roomserver"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// GET /rooms/{roomID}/relations/{eventID} and GET /rooms/{roomID}/threads:
// the children of an event, and the threads of a room.
//
// The reads are the roomserver's (relations.go there); what is here is the
// token handling, which is this package's. Relations page exactly like
// /messages and take the same tokens — a client may start paging an
// event's children from a sync's prev_batch — so they use the same
// stream-position token. Threads are ordered by their latest activity
// rather than by any one stream, and have a token of their own.
// DefaultRelationsLimit is the page size of /relations and /threads when
// the client names none.
const DefaultRelationsLimit = 50
// RelationsRequest is one page of an event's children.
type RelationsRequest struct {
RoomID string
EventID string
User string
RelType string
EventType string
// From and To are pagination tokens, as for /messages.
From string
To string
// Direction defaults to backwards, newest first, as the spec has it.
Direction storage.TimelineDirection
Limit int
Recurse bool
}
// RelationsResponse is one page of an event's children.
type RelationsResponse struct {
Chunk []roomserver.ClientEvent `json:"chunk"`
// NextBatch continues in the direction asked for; it is left out at
// the end.
NextBatch string `json:"next_batch,omitempty"`
// PrevBatch is the token this page began at, when the client gave one.
PrevBatch string `json:"prev_batch,omitempty"`
// RecursionDepth is how deep a recursive read went. It is only there
// when the client asked to recurse, which is how it learns the server
// honoured the request.
RecursionDepth int `json:"recursion_depth,omitempty"`
}
// Relations reads one page of an event's children.
func (s *API) Relations(ctx context.Context, req RelationsRequest) (RelationsResponse, error) {
from, err := ParseToken(req.From)
if err != nil {
return RelationsResponse{}, errInvalidTokenf("from: %v", err)
}
to, err := ParseToken(req.To)
if err != nil {
return RelationsResponse{}, errInvalidTokenf("to: %v", err)
}
direction := req.Direction
if direction != storage.DirectionForward {
direction = storage.DirectionBackward
}
limit := req.Limit
if limit <= 0 {
limit = DefaultRelationsLimit
}
limit = min(limit, MaxTimelineLimit)
low, high := from.PDU, to.PDU
if direction == storage.DirectionBackward {
low, high = to.PDU, from.PDU
}
page, err := s.rooms.Relations(ctx, roomserver.RelationsRequest{
RoomID: req.RoomID,
EventID: req.EventID,
User: req.User,
RelType: req.RelType,
EventType: req.EventType,
Low: low,
High: high,
Direction: direction,
Limit: limit,
Recurse: req.Recurse,
})
if err != nil {
return RelationsResponse{}, err
}
chunk := page.Events
if direction == storage.DirectionBackward {
chunk = reversed(chunk)
}
if chunk == nil {
chunk = []roomserver.ClientEvent{}
}
out := RelationsResponse{Chunk: chunk}
if req.From != "" {
out.PrevBatch = req.From
}
if page.Limited {
if direction == storage.DirectionBackward {
out.NextBatch = pduToken(page.Start).String()
} else {
out.NextBatch = pduToken(page.End).String()
}
}
if req.Recurse {
out.RecursionDepth = page.Depth
}
return out, nil
}
// ThreadsRequest is one page of a room's threads.
type ThreadsRequest struct {
RoomID string
User string
// Include is "all" or "participated".
Include string
// From is a token from a previous page's next_batch.
From string
Limit int
}
// ThreadsResponse is one page of a room's threads.
type ThreadsResponse struct {
Chunk []roomserver.ClientEvent `json:"chunk"`
NextBatch string `json:"next_batch,omitempty"`
}
// threadTokenPrefix marks a /threads token, which is a position in thread
// activity rather than a sync token.
const threadTokenPrefix = "t"
// Threads reads one page of a room's threads.
func (s *API) Threads(ctx context.Context, req ThreadsRequest) (ThreadsResponse, error) {
var participated bool
switch req.Include {
case "", "all":
case "participated":
participated = true
default:
return ThreadsResponse{}, errInvalidParamf("include must be \"all\" or \"participated\"")
}
var before int64
if req.From != "" {
raw, ok := strings.CutPrefix(req.From, threadTokenPrefix)
value, err := strconv.ParseInt(raw, 10, 64)
if !ok || err != nil {
return ThreadsResponse{}, errInvalidTokenf("%q is not a /threads token", req.From)
}
before = value
}
limit := req.Limit
if limit <= 0 {
limit = DefaultRelationsLimit
}
limit = min(limit, MaxTimelineLimit)
page, err := s.rooms.Threads(ctx, roomserver.ThreadsRequest{
RoomID: req.RoomID,
User: req.User,
Participated: participated,
Before: before,
Limit: limit,
})
if err != nil {
return ThreadsResponse{}, err
}
out := ThreadsResponse{Chunk: page.Roots}
if out.Chunk == nil {
out.Chunk = []roomserver.ClientEvent{}
}
if page.Next != 0 {
out.NextBatch = fmt.Sprintf("%s%d", threadTokenPrefix, page.Next)
}
return out, nil
}
+362
View File
@@ -0,0 +1,362 @@
package syncapi
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"github.com/dariuszjaniak/warp/server/internal/roomserver"
"github.com/dariuszjaniak/warp/server/internal/storage"
)
// POST /search: full-text search of the rooms a user is or was in.
//
// The roomserver decides where the user may look and the store runs the
// query; what is here is the request and response shape, which carries a
// fair amount besides the hits — the events around each one, the profiles of
// their senders, the rooms' state, and the hits grouped by room or sender —
// all of it optional and all of it what a client needs to show a result
// without a second round of requests.
// DefaultSearchLimit is the page size when the filter names none.
const DefaultSearchLimit = 10
// MaxSearchLimit caps a page. Every hit can bring its context, its
// senders' profiles and its room's state with it, so a page is far heavier
// than its count suggests.
const MaxSearchLimit = 100
// MaxSearchContext caps before_limit and after_limit.
const MaxSearchContext = 20
// SearchRequest is the body of a /search, with the caller.
type SearchRequest struct {
User string
// NextBatch is the ?next_batch= of a later page.
NextBatch string
Body SearchBody
}
// SearchBody is the request body.
type SearchBody struct {
SearchCategories struct {
RoomEvents *RoomEventsCriteria `json:"room_events"`
} `json:"search_categories"`
}
// RoomEventsCriteria is the one search category the spec defines.
type RoomEventsCriteria struct {
SearchTerm string `json:"search_term"`
Keys []string `json:"keys,omitempty"`
Filter *RoomEventFilter `json:"filter,omitempty"`
OrderBy string `json:"order_by,omitempty"`
EventContext *struct {
BeforeLimit *int `json:"before_limit,omitempty"`
AfterLimit *int `json:"after_limit,omitempty"`
IncludeProfile bool `json:"include_profile,omitempty"`
} `json:"event_context,omitempty"`
IncludeState bool `json:"include_state,omitempty"`
Groupings *struct {
GroupBy []struct {
Key string `json:"key"`
} `json:"group_by"`
} `json:"groupings,omitempty"`
}
// SearchResponse is the body of the answer.
type SearchResponse struct {
SearchCategories struct {
RoomEvents RoomEventsResults `json:"room_events"`
} `json:"search_categories"`
}
// RoomEventsResults is one page of hits.
type RoomEventsResults struct {
Count int `json:"count"`
Highlights []string `json:"highlights"`
Results []SearchHit `json:"results"`
NextBatch string `json:"next_batch,omitempty"`
State map[string][]roomserver.ClientEvent `json:"state,omitempty"`
Groups map[string]map[string]SearchGroupEntry `json:"groups,omitempty"`
}
// SearchHit is one result.
type SearchHit struct {
Rank float64 `json:"rank"`
Result roomserver.ClientEvent `json:"result"`
Context *SearchContext `json:"context,omitempty"`
}
// SearchContext is what surrounds one hit.
type SearchContext struct {
Start string `json:"start,omitempty"`
End string `json:"end,omitempty"`
EventsBefore []roomserver.ClientEvent `json:"events_before"`
EventsAfter []roomserver.ClientEvent `json:"events_after"`
ProfileInfo map[string]SearchProfileInfo `json:"profile_info,omitempty"`
}
// SearchProfileInfo is a sender's name and avatar.
type SearchProfileInfo struct {
DisplayName string `json:"displayname,omitempty"`
AvatarURL string `json:"avatar_url,omitempty"`
}
// SearchGroupEntry is one group of a grouped search.
type SearchGroupEntry struct {
Results []string `json:"results"`
Order int `json:"order"`
NextBatch string `json:"next_batch,omitempty"`
}
// The search tokens. A ranked search pages by offset, because the ranking
// of a result does not move as later events arrive; a search by recency
// pages by stream position, which keeps a page from repeating a result when
// new messages land between two requests.
const (
rankTokenPrefix = "r"
recentTokenPrefix = "p"
)
// Search answers one /search.
func (s *API) Search(ctx context.Context, req SearchRequest) (SearchResponse, error) {
criteria := req.Body.SearchCategories.RoomEvents
var resp SearchResponse
resp.SearchCategories.RoomEvents.Highlights = []string{}
resp.SearchCategories.RoomEvents.Results = []SearchHit{}
if criteria == nil {
// The only category there is was not asked for, so there is
// nothing to search.
return resp, nil
}
if strings.TrimSpace(criteria.SearchTerm) == "" {
return SearchResponse{}, errInvalidParamf("search_term must not be empty")
}
order := storage.SearchByRank
switch criteria.OrderBy {
case "", "rank":
case "recent":
order = storage.SearchByRecent
default:
return SearchResponse{}, errInvalidParamf("order_by must be \"rank\" or \"recent\"")
}
for _, key := range criteria.Keys {
switch key {
case "content.body", "content.name", "content.topic":
default:
return SearchResponse{}, errInvalidParamf("%q is not a search key", key)
}
}
offset, before, err := parseSearchToken(req.NextBatch, order)
if err != nil {
return SearchResponse{}, err
}
limit := DefaultSearchLimit
var filter storage.TimelineFilter
var rooms, notRooms []string
if f := criteria.Filter; f != nil {
if f.Limit != nil && *f.Limit > 0 {
limit = min(*f.Limit, MaxSearchLimit)
}
filter = roomEventFilter(f)
rooms, notRooms = f.Rooms, f.NotRooms
}
found, total, err := s.rooms.Search(ctx, roomserver.SearchRequest{
User: req.User,
Term: criteria.SearchTerm,
Keys: criteria.Keys,
Rooms: rooms,
NotRooms: notRooms,
Filter: filter,
Order: order,
Offset: offset,
Before: before,
Limit: limit,
})
if err != nil {
return SearchResponse{}, err
}
out := &resp.SearchCategories.RoomEvents
out.Count = total
out.Highlights = storage.SearchWords(criteria.SearchTerm)
for _, hit := range found {
h := SearchHit{Rank: hit.Rank, Result: hit.Event}
if criteria.EventContext != nil {
if h.Context, err = s.searchContext(ctx, req.User, hit, criteria); err != nil {
return SearchResponse{}, err
}
}
out.Results = append(out.Results, h)
}
// There is more when the page came back full and the total says so.
if len(found) == limit {
switch order {
case storage.SearchByRank:
if offset+len(found) < total {
out.NextBatch = rankTokenPrefix + strconv.Itoa(offset+len(found))
}
case storage.SearchByRecent:
out.NextBatch = recentTokenPrefix + strconv.FormatInt(found[len(found)-1].Position, 10)
}
}
if criteria.IncludeState {
out.State = map[string][]roomserver.ClientEvent{}
for _, hit := range found {
roomID := hit.Event.RoomID
if _, done := out.State[roomID]; done {
continue
}
state, err := s.rooms.RoomState(ctx, roomID, req.User)
if err != nil {
return SearchResponse{}, err
}
out.State[roomID] = state
}
}
if criteria.Groupings != nil {
out.Groups = searchGroups(found, criteria, out.NextBatch)
}
return resp, nil
}
// searchContext reads the events around one hit.
func (s *API) searchContext(ctx context.Context, userID string, hit roomserver.SearchResult, criteria *RoomEventsCriteria) (*SearchContext, error) {
limitOf := func(p *int) int {
if p == nil {
return 5
}
return max(0, min(*p, MaxSearchContext))
}
beforeLimit := limitOf(criteria.EventContext.BeforeLimit)
afterLimit := limitOf(criteria.EventContext.AfterLimit)
roomID := hit.Event.RoomID
out := &SearchContext{
EventsBefore: []roomserver.ClientEvent{},
EventsAfter: []roomserver.ClientEvent{},
}
if beforeLimit > 0 {
earlier, err := s.rooms.Timeline(ctx, roomserver.TimelineRequest{
RoomID: roomID,
User: userID,
High: storage.PositionBefore(hit.Position),
Direction: storage.DirectionBackward,
Limit: beforeLimit,
})
if err != nil {
return nil, err
}
if len(earlier.Events) > 0 {
out.EventsBefore = reversed(earlier.Events)
out.Start = pduToken(earlier.Start).String()
}
}
if afterLimit > 0 {
later, err := s.rooms.Timeline(ctx, roomserver.TimelineRequest{
RoomID: roomID,
User: userID,
Low: hit.Position,
Direction: storage.DirectionForward,
Limit: afterLimit,
})
if err != nil {
return nil, err
}
if len(later.Events) > 0 {
out.EventsAfter = later.Events
out.End = pduToken(later.End).String()
}
}
if criteria.EventContext.IncludeProfile {
senders := sendersOf(append(append([]roomserver.ClientEvent{hit.Event}, out.EventsBefore...), out.EventsAfter...))
members, err := s.rooms.MemberEvents(ctx, roomID, senders)
if err != nil {
return nil, err
}
out.ProfileInfo = map[string]SearchProfileInfo{}
for _, m := range members {
if m.StateKey == nil {
continue
}
var content struct {
DisplayName string `json:"displayname"`
AvatarURL string `json:"avatar_url"`
}
_ = json.Unmarshal(m.Content, &content)
out.ProfileInfo[*m.StateKey] = SearchProfileInfo{DisplayName: content.DisplayName, AvatarURL: content.AvatarURL}
}
}
return out, nil
}
// searchGroups groups a page of hits by room or sender, in the order the
// page ranked them.
func searchGroups(found []roomserver.SearchResult, criteria *RoomEventsCriteria, nextBatch string) map[string]map[string]SearchGroupEntry {
out := map[string]map[string]SearchGroupEntry{}
for _, g := range criteria.Groupings.GroupBy {
var keyOf func(roomserver.SearchResult) string
switch g.Key {
case "room_id":
keyOf = func(r roomserver.SearchResult) string { return r.Event.RoomID }
case "sender":
keyOf = func(r roomserver.SearchResult) string { return r.Event.Sender }
default:
continue
}
groups := map[string]SearchGroupEntry{}
for _, hit := range found {
key := keyOf(hit)
entry, ok := groups[key]
if !ok {
entry.Order = len(groups) + 1
entry.NextBatch = nextBatch
}
entry.Results = append(entry.Results, hit.Event.EventID)
groups[key] = entry
}
out[g.Key] = groups
}
return out
}
// parseSearchToken reads a ?next_batch=.
func parseSearchToken(raw string, order storage.SearchOrder) (offset int, before int64, err error) {
if raw == "" {
return 0, 0, nil
}
bad := errInvalidTokenf("%q is not a search token", raw)
switch order {
case storage.SearchByRecent:
body, ok := strings.CutPrefix(raw, recentTokenPrefix)
if !ok {
return 0, 0, bad
}
if before, err = strconv.ParseInt(body, 10, 64); err != nil {
return 0, 0, bad
}
return 0, before, nil
default:
body, ok := strings.CutPrefix(raw, rankTokenPrefix)
if !ok {
return 0, 0, bad
}
if offset, err = strconv.Atoi(body); err != nil || offset < 0 {
return 0, 0, fmt.Errorf("%w", bad)
}
return offset, 0, nil
}
}
File diff suppressed because it is too large. Load diff
+219
View File
@@ -0,0 +1,219 @@
package syncapi
import (
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"maps"
"slices"
"strconv"
"strings"
"sync"
"time"
)
// Sliding sync connections: what the server remembers about what each
// client has already been sent.
//
// A classic sync token is a position in every stream and nothing else; the
// client is sent everything after it. Sliding sync cannot work that way,
// because what a client is sent depends on what it asked for — which rooms
// are in the window of which list, with which pieces of state — and that
// changes from request to request. So the server remembers, per connection,
// which rooms it has sent and as of which position, and `pos` names one of
// those memories.
//
// The memory is in process and is lost on restart. That is what the
// protocol expects: an unknown `pos` is answered with M_UNKNOWN_POS, and
// the client starts a new connection — one initial request, after which it
// is in step again. Persisting it would buy a slightly cheaper restart for
// a write on every request.
// ErrUnknownPos is a `pos` this server does not remember: expired, from
// before a restart, or never handed out.
var ErrUnknownPos = errors.New("unknown sliding sync position")
// Limits of the connection store.
const (
// connectionTTL is how long an idle connection is remembered. A client
// that comes back later starts again.
connectionTTL = 30 * time.Minute
// maxConnectionsPerDevice caps the connections one device may hold
// open, each conn_id being one. The oldest goes first.
maxConnectionsPerDevice = 10
// keptGenerations is how many of a connection's latest positions stay
// usable. A client that never saw a response retries with the pos it
// sent, so the one before the latest has to still work.
keptGenerations = 3
)
// sentRoom is what a connection has been sent of one room.
type sentRoom struct {
// pos is the event stream position the room was last sent up to.
pos int64
// config is the room's required_state and timeline_limit as of that
// send, rendered as a string: when it changes, the state the client
// holds is no longer what it asked for.
config string
// membership is the user's membership as of that send.
membership string
// typing is the typing notification last sent, as its JSON.
typing string
}
// connSnapshot is a connection as of one `pos`.
type connSnapshot struct {
// token is the stream positions the response at this pos read up to.
token Token
// startPDU is the event stream position the connection began at: rooms
// the user left before it are not news to it.
startPDU int64
rooms map[string]sentRoom
// lists is each list's room count as last sent.
lists map[string]int
// toDevice is where the to_device extension has got to.
toDevice int64
}
// clone copies a snapshot so that a new generation can be built from it
// without touching the one a retry might still need.
func (c *connSnapshot) clone() *connSnapshot {
return &connSnapshot{
token: c.token,
startPDU: c.startPDU,
rooms: maps.Clone(c.rooms),
lists: maps.Clone(c.lists),
toDevice: c.toDevice,
}
}
// slidingConn is one connection: a client device's conn_id.
type slidingConn struct {
// nonce makes the connection's positions its own: a pos from another
// connection, or from before a restart, cannot name one of these
// generations by accident.
nonce string
generations map[int64]*connSnapshot
latest int64
lastUsed time.Time
}
type connKey struct {
user, device, connID string
}
// connStore holds every connection.
type connStore struct {
mu sync.Mutex
conns map[connKey]*slidingConn
now func() time.Time
}
func newConnStore(now func() time.Time) *connStore {
return &connStore{conns: map[connKey]*slidingConn{}, now: now}
}
// load returns the snapshot a request's pos names, or a fresh one for a
// request with none. A request without a pos starts the connection over,
// which is what a client does after M_UNKNOWN_POS and on first launch.
func (s *connStore) load(key connKey, pos string) (*connSnapshot, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.sweep()
if pos == "" {
delete(s.conns, key)
return nil, nil
}
conn, ok := s.conns[key]
if !ok {
return nil, fmt.Errorf("%w: the connection has expired", ErrUnknownPos)
}
nonce, raw, ok := strings.Cut(pos, ".")
gen, err := strconv.ParseInt(raw, 10, 64)
if !ok || err != nil || nonce != conn.nonce {
return nil, fmt.Errorf("%w: %q is not a position of this connection", ErrUnknownPos, pos)
}
snap, ok := conn.generations[gen]
if !ok {
return nil, fmt.Errorf("%w: %q is too old", ErrUnknownPos, pos)
}
conn.lastUsed = s.now()
return snap, nil
}
// save stores a new generation of a connection and returns its pos.
func (s *connStore) save(key connKey, snap *connSnapshot) string {
s.mu.Lock()
defer s.mu.Unlock()
conn, ok := s.conns[key]
if !ok {
conn = &slidingConn{nonce: randomNonce(), generations: map[int64]*connSnapshot{}, lastUsed: s.now()}
s.conns[key] = conn
s.capDevice(key)
}
conn.latest++
conn.generations[conn.latest] = snap
delete(conn.generations, conn.latest-keptGenerations)
conn.lastUsed = s.now()
return conn.nonce + "." + strconv.FormatInt(conn.latest, 10)
}
// sweep forgets idle connections. It runs under the lock on every load,
// which keeps the store bounded without a goroutine of its own.
func (s *connStore) sweep() {
cutoff := s.now().Add(-connectionTTL)
for key, conn := range s.conns {
if conn.lastUsed.Before(cutoff) {
delete(s.conns, key)
}
}
}
// capDevice drops a device's oldest connections beyond the cap. The
// connection being saved is never one of them.
func (s *connStore) capDevice(key connKey) {
var others []connKey
for k := range s.conns {
if k != key && k.user == key.user && k.device == key.device {
others = append(others, k)
}
}
slices.SortFunc(others, func(a, b connKey) int {
return s.conns[a].lastUsed.Compare(s.conns[b].lastUsed)
})
for len(others) > maxConnectionsPerDevice-1 {
delete(s.conns, others[0])
others = others[1:]
}
}
// connections reports how many connections are held, for tests and
// metrics.
func (s *connStore) connections() int {
s.mu.Lock()
defer s.mu.Unlock()
return len(s.conns)
}
func randomNonce() string {
var b [6]byte
_, _ = rand.Read(b[:])
return hex.EncodeToString(b[:])
}
// SlidingConnections reports how many sliding sync connections the server
// is holding state for.
func (s *API) SlidingConnections() int { return s.conns.connections() }
+34 -9
View File
@@ -7,9 +7,12 @@ import (
"slices"
"time"
"go.opentelemetry.io/otel/attribute"
"github.com/dariuszjaniak/warp/server/internal/matrix/event"
"github.com/dariuszjaniak/warp/server/internal/roomserver"
"github.com/dariuszjaniak/warp/server/internal/storage"
"github.com/dariuszjaniak/warp/server/internal/tracing"
)
// GET /sync: the endpoint a client spends its life in.
@@ -103,6 +106,12 @@ type JoinedRoom struct {
// it out rather than claiming zero: an absent count says nothing,
// where a zero says "you have read everything".
UnreadNotifications *UnreadNotifications `json:"unread_notifications,omitempty"`
// UnreadThreadNotifications is the counts of each thread with
// something unread, keyed by the thread root's event ID, for a filter
// that asks for them; UnreadNotifications then counts the main
// timeline alone.
UnreadThreadNotifications map[string]UnreadNotifications `json:"unread_thread_notifications,omitempty"`
}
// InvitedRoom is a room the user has been invited to. It carries no
@@ -266,7 +275,10 @@ func (s *API) Sync(ctx context.Context, req SyncRequest) (Response, error) {
// blocking on a change it has just missed. The listener is returned rather
// than closed here because the caller is the one that knows whether it is
// about to wait on it.
func (s *API) attempt(ctx context.Context, req SyncRequest, filter Filter, since Token) (Response, bool, *Listener, error) {
func (s *API) attempt(ctx context.Context, req SyncRequest, filter Filter, since Token) (_ Response, _ bool, _ *Listener, err error) {
ctx, span := tracing.Start(ctx, "syncapi.sync", attribute.Bool("matrix.initial", since.IsInitial()))
defer func() { tracing.End(span, err) }()
rooms, err := s.rooms.UserMemberships(ctx, req.User, nil)
if err != nil {
return Response{}, false, nil, err
@@ -333,17 +345,18 @@ func (s *API) build(ctx context.Context, req SyncRequest, filter Filter, since T
if err := s.buildRooms(ctx, req, filter, since, to, rooms, &resp); err != nil {
return Response{}, false, err
}
if err := s.unreadNotifications(ctx, req.User, &resp); err != nil {
if err := s.unreadNotifications(ctx, req.User, filter.UnreadThreadNotifications(), &resp); err != nil {
return Response{}, false, err
}
accountData, err := s.globalAccountData(ctx, req, since.AccountData, to.AccountData, filter)
accountData, err := s.globalAccountData(ctx, req, since.AccountData, to.AccountData,
req.Since == "" || req.FullState, filter)
if err != nil {
return Response{}, false, err
}
resp.AccountData.Events = accountData
toDevice, toDevicePos, err := s.toDeviceMessages(ctx, req.Localpart, req.Device, since.ToDevice, to.ToDevice)
toDevice, toDevicePos, err := s.toDeviceMessages(ctx, req.Localpart, req.Device, since.ToDevice, to.ToDevice, 0)
if err != nil {
return Response{}, false, err
}
@@ -373,7 +386,7 @@ func (s *API) build(ctx context.Context, req SyncRequest, filter Filter, since T
}
resp.NextBatch = next.String()
return resp, isInteresting(resp, since), nil
return resp, isInteresting(resp, req.Since == ""), nil
}
// isInteresting reports whether a response is worth returning rather than
@@ -385,8 +398,13 @@ func (s *API) build(ctx context.Context, req SyncRequest, filter Filter, since T
// into a busy loop. So the test is whether the response has anything in it,
// with an initial sync as the exception: a client that has just logged in
// needs an answer whatever is in it.
func isInteresting(resp Response, since Token) bool {
if since.IsInitial() {
//
// Initial means the client sent no ?since=, not that the token it sent is
// all zeroes. On a server whose streams have not moved yet — no events, no
// receipts, nothing — "s0" is a client's every token, and a sync that took
// that for a first one would never wait.
func isInteresting(resp Response, initial bool) bool {
if initial {
return true
}
return len(resp.Rooms.Join) > 0 ||
@@ -665,7 +683,8 @@ func (s *API) roomEphemeral(ctx context.Context, userID, roomID string, window r
}
// globalAccountData reads the account data that is not about a room.
func (s *API) globalAccountData(ctx context.Context, req SyncRequest, from, to int64, filter Filter) ([]AccountDataEvent, error) {
// initial is whether this is the client's first read of it.
func (s *API) globalAccountData(ctx context.Context, req SyncRequest, from, to int64, initial bool, filter Filter) ([]AccountDataEvent, error) {
entries, err := s.accountDataEntries(ctx, req.Localpart, from, to)
if err != nil {
return nil, err
@@ -673,7 +692,13 @@ func (s *API) globalAccountData(ctx context.Context, req SyncRequest, from, to i
// Every user has push rules, written or not: a user who has never
// changed one has the server defaults, and a client reads its rules
// from the account data of its first sync whichever it is.
if from == 0 && s.push != nil && !slices.ContainsFunc(entries, func(e storage.AccountDataEntry) bool {
//
// "First" is the client's first sync, not a token whose account data
// position is zero: on a server where nobody has written any account
// data yet every token has a zero there, and treating each sync as the
// first would send the rules every time — which makes every sync
// "interesting", and turns every long poll into a busy loop.
if initial && s.push != nil && !slices.ContainsFunc(entries, func(e storage.AccountDataEntry) bool {
return e.RoomID == "" && e.DataType == pushRulesType
}) {
entries = append(entries, storage.AccountDataEntry{DataType: pushRulesType})
+9 -3
View File
@@ -84,12 +84,14 @@ type Keys interface {
}
// Push is the part of the push server a sync reads: the push rules it
// reports as account data, the unread counts of each room, and which
// reports as account data, the unread counts of each room (and of each
// thread in it, for a client that asks), and which
// rooms' counts moved. It is also told about read receipts, which are what
// clear the counts. *pushserver.API satisfies it.
type Push interface {
Document(ctx context.Context, localpart, userID string) ([]byte, error)
NotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]storage.NotificationCounts, error)
ThreadNotificationCounts(ctx context.Context, userID string, roomIDs []string) (map[string]map[string]storage.NotificationCounts, error)
NotificationCountChanges(ctx context.Context, userID string, from, to int64) ([]string, error)
ReceiptSet(ctx context.Context, userID, roomID, receiptType, threadID, eventID string)
}
@@ -104,8 +106,11 @@ type API struct {
store storage.SyncStore
notifier *Notifier
typing *typingTracker
log *slog.Logger
now func() time.Time
// conns remembers what each sliding sync connection has been sent.
conns *connStore
log *slog.Logger
now func() time.Time
// presenceEnabled is the config switch of PLAN.md open question 3.
// With presence off the endpoints answer M_FORBIDDEN and the presence
@@ -227,6 +232,7 @@ func New(opts Options) (*API, error) {
store: opts.Store,
notifier: opts.Notifier,
typing: newTypingTracker(opts.TypingTimeout, opts.Notifier, opts.Now),
conns: newConnStore(opts.Now),
log: opts.Logger,
now: opts.Now,
presenceEnabled: opts.PresenceEnabled,
+2 -2
View File
@@ -140,14 +140,14 @@ type toDeviceEvent struct {
// position of the last message *delivered* rather than the top of the
// stream: a queue longer than one batch is drained over several syncs, and
// a token naming the top would skip the rest.
func (s *API) toDeviceMessages(ctx context.Context, localpart, deviceID string, from, to int64) ([]toDeviceEvent, int64, error) {
func (s *API) toDeviceMessages(ctx context.Context, localpart, deviceID string, from, to int64, limit int) ([]toDeviceEvent, int64, error) {
if deviceID == "" {
// An access token always names a device, so this cannot happen
// through the API; a guest session later might.
return nil, from, nil
}
msgs, err := s.store.ToDeviceMessages(ctx, localpart, deviceID, from, to, 0)
msgs, err := s.store.ToDeviceMessages(ctx, localpart, deviceID, from, to, limit)
if err != nil {
return nil, 0, err
}
+199
View File
@@ -0,0 +1,199 @@
// Package tracing is Warp's OpenTelemetry tracing: a span per HTTP
// request, spans around the work that request does in the components
// (persisting an event, building a sync, calling another server), and the
// W3C trace context passed on to the servers and services Warp calls, so
// that a federation request or an application service transaction shows up
// in the same trace as the client request that caused it.
//
// Tracing is off unless tracing.enabled is set. Off, the global tracer
// provider is OpenTelemetry's no-op one, and every span below costs a
// function call that returns at once — which is what lets the components
// start spans unconditionally rather than checking a flag.
//
// Spans are exported over OTLP/HTTP, which every collector and every
// tracing backend worth running (Jaeger, Tempo, Honeycomb, an OpenTelemetry
// Collector in front of anything else) accepts.
package tracing
import (
"context"
"errors"
"fmt"
"net/http"
"net/url"
"strconv"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.43.0"
"go.opentelemetry.io/otel/trace"
"github.com/dariuszjaniak/warp/server/internal/httputil"
)
// instrumentation is the name Warp's spans are reported under.
const instrumentation = "github.com/dariuszjaniak/warp/server"
// Config is what Setup needs.
type Config struct {
Enabled bool
// Endpoint is the OTLP/HTTP collector, e.g. "http://localhost:4318".
// Empty leaves it to the OTEL_EXPORTER_OTLP_* environment variables,
// and to the exporter's default of localhost:4318 without them.
Endpoint string
// SampleRatio is the fraction of traces started here that are kept,
// from 0 to 1. A request that arrives with a sampled parent is always
// kept, whatever the ratio: the caller has already decided.
SampleRatio float64
ServiceName string
ServiceVersion string
// ServerName is the Matrix server name, recorded on every span so that
// traces from several homeservers sharing a collector can be told
// apart.
ServerName string
}
// Setup installs the global tracer provider and propagator, and returns
// the function that flushes and stops them.
func Setup(ctx context.Context, cfg Config) (func(context.Context) error, error) {
// The propagator is installed even with tracing off: a Warp that does
// not record still passes on the trace context it was given, so that a
// trace through it is not broken in two.
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{}, propagation.Baggage{}))
if !cfg.Enabled {
return func(context.Context) error { return nil }, nil
}
var opts []otlptracehttp.Option
if cfg.Endpoint != "" {
u, err := url.Parse(cfg.Endpoint)
if err != nil || u.Host == "" {
return nil, fmt.Errorf("tracing: endpoint %q is not a URL", cfg.Endpoint)
}
opts = append(opts, otlptracehttp.WithEndpointURL(cfg.Endpoint))
}
exporter, err := otlptracehttp.New(ctx, opts...)
if err != nil {
return nil, fmt.Errorf("tracing: build the OTLP exporter: %w", err)
}
name := cfg.ServiceName
if name == "" {
name = "warp"
}
res, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL,
semconv.ServiceName(name),
semconv.ServiceVersion(cfg.ServiceVersion),
attribute.String("matrix.server_name", cfg.ServerName),
))
if err != nil && !errors.Is(err, resource.ErrSchemaURLConflict) {
return nil, fmt.Errorf("tracing: describe the service: %w", err)
}
provider := sdktrace.NewTracerProvider(
sdktrace.WithBatcher(exporter),
sdktrace.WithResource(res),
sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRatio))),
)
otel.SetTracerProvider(provider)
return provider.Shutdown, nil
}
// Tracer is the tracer Warp's components start spans with.
func Tracer() trace.Tracer { return otel.Tracer(instrumentation) }
// Start begins a span in the current trace. It is a no-op span when
// tracing is off.
func Start(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, trace.Span) {
return Tracer().Start(ctx, name, trace.WithAttributes(attrs...))
}
// End finishes a span, recording err on it when there is one.
func End(span trace.Span, err error) {
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
}
span.End()
}
// Instrument wraps one route's handler in a server span named after the
// route's pattern — "PUT /_matrix/client/v3/rooms/{roomID}/send/..." —
// rather than its path, which would make every room its own span name.
//
// It has the shape of httputil.Instrumenter, so it composes with the
// metrics middleware at route registration.
func Instrument(route string, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := otel.GetTextMapPropagator().Extract(r.Context(), propagation.HeaderCarrier(r.Header))
ctx, span := Tracer().Start(ctx, route,
trace.WithSpanKind(trace.SpanKindServer),
trace.WithAttributes(
semconv.HTTPRequestMethodKey.String(r.Method),
semconv.HTTPRoute(route),
semconv.URLPath(r.URL.Path),
semconv.UserAgentOriginal(r.UserAgent()),
))
defer span.End()
rec := &httputil.ResponseRecorder{ResponseWriter: w}
next.ServeHTTP(rec, r.WithContext(ctx))
status := rec.Status
if status == 0 {
status = http.StatusOK
}
span.SetAttributes(semconv.HTTPResponseStatusCode(status))
if status >= http.StatusInternalServerError {
span.SetStatus(codes.Error, strconv.Itoa(status))
}
})
}
// Transport wraps an outbound HTTP transport so that every request it sends
// is a client span, carrying the trace context to the other side. A nil
// base is http.DefaultTransport.
func Transport(base http.RoundTripper) http.RoundTripper {
if base == nil {
base = http.DefaultTransport
}
return &transport{base: base}
}
type transport struct{ base http.RoundTripper }
func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) {
ctx, span := Tracer().Start(req.Context(), req.Method+" "+req.URL.Host,
trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(
semconv.HTTPRequestMethodKey.String(req.Method),
semconv.ServerAddress(req.URL.Hostname()),
semconv.URLPath(req.URL.Path),
))
defer span.End()
// RoundTrip must not modify the request it was given.
out := req.Clone(ctx)
otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(out.Header))
resp, err := t.base.RoundTrip(out)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return nil, err
}
span.SetAttributes(semconv.HTTPResponseStatusCode(resp.StatusCode))
if resp.StatusCode >= http.StatusInternalServerError {
span.SetStatus(codes.Error, strconv.Itoa(resp.StatusCode))
}
return resp, nil
}
+122
View File
@@ -0,0 +1,122 @@
package tracing_test
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"go.opentelemetry.io/otel"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
"go.opentelemetry.io/otel/trace"
"github.com/dariuszjaniak/warp/server/internal/tracing"
)
// recordSpans installs a tracer provider that keeps every span in memory.
func recordSpans(t *testing.T) *tracetest.InMemoryExporter {
t.Helper()
if _, err := tracing.Setup(t.Context(), tracing.Config{}); err != nil {
t.Fatalf("Setup: %v", err)
}
exporter := tracetest.NewInMemoryExporter()
provider := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter))
previous := otel.GetTracerProvider()
otel.SetTracerProvider(provider)
t.Cleanup(func() {
_ = provider.Shutdown(context.Background())
otel.SetTracerProvider(previous)
})
return exporter
}
func TestInstrumentNamesTheSpanAfterTheRoute(t *testing.T) {
exporter := recordSpans(t)
const route = "GET /_matrix/client/v3/rooms/{roomID}/state"
var inner trace.SpanContext
h := tracing.Instrument(route, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
inner = trace.SpanContextFromContext(r.Context())
w.WriteHeader(http.StatusTeapot)
}))
// A request that arrives inside a trace continues it.
req := httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/rooms/!a:b/state", nil)
req.Header.Set("traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01")
h.ServeHTTP(httptest.NewRecorder(), req)
spans := exporter.GetSpans()
if len(spans) != 1 {
t.Fatalf("recorded %d spans", len(spans))
}
span := spans[0]
if span.Name != route {
t.Errorf("span name = %q", span.Name)
}
if span.SpanKind != trace.SpanKindServer {
t.Errorf("span kind = %v", span.SpanKind)
}
if got := span.SpanContext.TraceID().String(); got != "4bf92f3577b34da6a3ce929d0e0e4736" {
t.Errorf("trace ID = %s, want the caller's", got)
}
if !inner.IsValid() || inner.SpanID() != span.SpanContext.SpanID() {
t.Error("the handler did not run inside the span")
}
var status int64
for _, a := range span.Attributes {
if a.Key == "http.response.status_code" {
status = a.Value.AsInt64()
}
}
if status != http.StatusTeapot {
t.Errorf("status attribute = %d", status)
}
}
func TestTransportCarriesTheTraceOn(t *testing.T) {
exporter := recordSpans(t)
var traceparent string
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
traceparent = r.Header.Get("traceparent")
}))
defer upstream.Close()
ctx, parent := tracing.Start(t.Context(), "parent")
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, upstream.URL+"/x", nil)
resp, err := (&http.Client{Transport: tracing.Transport(nil)}).Do(req)
if err != nil {
t.Fatalf("request: %v", err)
}
_ = resp.Body.Close()
tracing.End(parent, nil)
if traceparent == "" {
t.Fatal("the outbound request carried no traceparent")
}
if req.Header.Get("traceparent") != "" {
t.Error("the caller's request was modified")
}
if spans := exporter.GetSpans(); len(spans) != 2 {
t.Errorf("recorded %d spans, want the client span and its parent", len(spans))
}
}
func TestDisabledTracingIsANoOp(t *testing.T) {
stop, err := tracing.Setup(t.Context(), tracing.Config{Enabled: false})
if err != nil {
t.Fatalf("Setup: %v", err)
}
_, span := tracing.Start(t.Context(), "nothing")
if span.SpanContext().IsSampled() {
t.Error("a span was sampled with tracing off")
}
tracing.End(span, nil)
if err := stop(t.Context()); err != nil {
t.Errorf("stop: %v", err)
}
if _, err := tracing.Setup(t.Context(), tracing.Config{Enabled: true, Endpoint: "not a url"}); err == nil {
t.Error("a malformed endpoint was accepted")
}
}