Files
Warp/server/internal/storage/sqlstore/sqlstore.go
T
tajniak81andClaude Opus 5.5 487e0f410b
CI / Test (ubuntu-latest) (push) Canceled after 0s
CI / Test (windows-latest) (push) Canceled after 0s
CI / Storage tests on PostgreSQL (push) Canceled after 0s
CI / Complement (federation off) (push) Canceled after 0s
CI / golangci-lint (push) Canceled after 0s
CI / Smoke test the binary (push) Canceled after 0s
Phase 10: hardening and ops
Implements PLAN.md Phase 10.

Simplified Sliding Sync (MSC4186) is served at the unstable
org.matrix.simplified_msc3575 endpoint and advertised in /versions. Lists
are sorted by activity and filtered by DM, invite, encryption, room type,
space and tag; room subscriptions, required_state with wildcards, $ME and
$LAZY, and the to_device, e2ee, account_data, receipts and typing
extensions are supported. Per-connection state is kept in memory and an
unknown pos answers M_UNKNOWN_POS.

Threads and relations: m.relates_to is indexed on roomserver_events
(migration 0012), /relations (with recursion) and /threads are served, and
every event a timeline read returns carries its m.thread, m.replace and
m.reference summaries in unsigned.m.relations. Notifications carry a
thread ID (migration 0013): threaded receipts clear only their thread, and
sync reports unread_thread_notifications when the filter asks.

Spaces: /hierarchy walks a space breadth first, asking other servers about
rooms this one is not in through the new federation hierarchy endpoint;
room_summary describes one room. /search runs on FTS5 (sqlite, pocketbase)
or tsvector (postgres), bounded to what each user may see, with context,
profiles, state, grouping and paging.

The PocketBase driver is real: the sqlite schema and store on an embedded
PocketBase app's database, with media in its filesystem, local or S3
(database.pocketbase.s3). It runs the full storage test suite.

Ops: OpenTelemetry tracing over OTLP/HTTP (tracing.*), with server spans
per route, component spans, and trace propagation on federation, push and
application service calls. New metrics for persisted events, waiting
syncs, sliding sync connections and the database pool, with a Grafana
dashboard. deploy/ adds a Compose setup (PostgreSQL, Prometheus, Grafana,
Jaeger) and a Helm chart; cmd/warp-loadtest loads a server and reports
request and delivery latencies.

The load tester found a long-standing bug, fixed here: on a server where
no account data had been written, every incremental /sync re-sent the push
rules and returned at once, and an all-zero token was taken for an initial
sync, so long polls became busy loops.

The PostgreSQL migrations and search dialect compile but were not run
against a live PostgreSQL in this change.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-17 20:21:31 +02:00

184 lines
5.4 KiB
Go

// Package sqlstore holds the parts of a storage driver that are identical
// for every database/sql backend: the migration plumbing and the
// server_settings table. The sqlite and postgres drivers differ only in
// their DSN handling, placeholder style and upsert syntax, all of which
// they pass in here.
package sqlstore
import (
"context"
"database/sql"
"errors"
"fmt"
"io/fs"
"log/slog"
"github.com/dariuszjaniak/warp/server/internal/storage"
"github.com/dariuszjaniak/warp/server/internal/storage/migrations"
)
// Options describe one database/sql backend.
type Options struct {
// DriverName is the Warp driver name, e.g. "sqlite".
DriverName string
// DB is the already-opened pool.
DB *sql.DB
// Dialect supplies the migration bookkeeping statements.
Dialect migrations.Dialect
// MigrationsFS and MigrationsDir locate the embedded .sql files.
MigrationsFS fs.FS
MigrationsDir string
// Settings holds the statements for the server_settings table.
Settings SettingsQueries
// Rebind translates the `?` placeholders that the shared component
// stores are written with into this driver's style. SQLite needs no
// translation and leaves it nil; PostgreSQL turns them into $1, $2...
Rebind func(query string) string
// IsUniqueViolation recognises this driver's unique-constraint error, so
// that a taken user name can be told apart from a database failure
// without a check-then-insert race.
IsUniqueViolation func(error) bool
// Search is this engine's full-text search. Nil leaves the index
// unwritten and /search finding nothing.
Search *SearchDialect
// Logger is used for migration progress.
Logger *slog.Logger
}
// Store is the shared implementation of storage.Database. Drivers embed it
// and add their own component stores as those are built.
type Store struct {
opts Options
db *sql.DB
log *slog.Logger
settings *Settings
users *Users
rooms *Rooms
sync *Sync
fed *Federation
media *Media
keys *Keys
push *Push
admin *Admin
apps *AppServices
}
// New builds a Store. It does not touch the database.
func New(opts Options) (*Store, error) {
if opts.DB == nil {
return nil, errors.New("sqlstore: DB must not be nil")
}
if opts.Logger == nil {
opts.Logger = slog.Default()
}
return &Store{
opts: opts,
db: opts.DB,
log: opts.Logger,
settings: NewSettings(opts.DB, opts.Settings),
users: NewUsers(opts.DB, opts.Rebind, opts.IsUniqueViolation),
rooms: NewRooms(opts.DB, opts.Rebind, opts.IsUniqueViolation).WithSearch(opts.Search),
sync: NewSync(opts.DB, opts.Rebind, opts.IsUniqueViolation),
fed: NewFederation(opts.DB, opts.Rebind),
media: NewMedia(opts.DB, opts.Rebind, opts.IsUniqueViolation),
keys: NewKeys(opts.DB, opts.Rebind),
push: NewPush(opts.DB, opts.Rebind),
admin: NewAdmin(opts.DB, opts.Rebind, opts.IsUniqueViolation),
apps: NewAppServices(opts.DB, opts.Rebind),
}, nil
}
// DB exposes the pool for driver-specific code and tests.
func (s *Store) DB() *sql.DB { return s.db }
// Driver implements storage.Database.
func (s *Store) Driver() string { return s.opts.DriverName }
// Ping implements storage.Database.
func (s *Store) Ping(ctx context.Context) error { return s.db.PingContext(ctx) }
// Close implements storage.Database.
func (s *Store) Close() error { return s.db.Close() }
// Settings implements storage.Database.
func (s *Store) Settings() storage.SettingsStore { return s.settings }
// Users implements storage.Database.
func (s *Store) Users() storage.UserStore { return s.users }
// Rooms implements storage.Database.
func (s *Store) Rooms() storage.RoomStore { return s.rooms }
// Sync implements storage.Database.
func (s *Store) Sync() storage.SyncStore { return s.sync }
// Federation implements storage.Database.
func (s *Store) Federation() storage.FederationStore { return s.fed }
// Media implements storage.Database.
func (s *Store) Media() storage.MediaStore { return s.media }
// Keys implements storage.Database.
func (s *Store) Keys() storage.KeyStore { return s.keys }
// Push implements storage.Database.
func (s *Store) Push() storage.PushStore { return s.push }
// Admin implements storage.Database.
func (s *Store) Admin() storage.AdminStore { return s.admin }
// AppServices implements storage.Database.
func (s *Store) AppServices() storage.AppServiceStore { return s.apps }
// Migrate implements storage.Database.
func (s *Store) Migrate(ctx context.Context) error {
ms, err := s.migrations()
if err != nil {
return err
}
applied, err := migrations.Status(ctx, s.db, s.opts.Dialect)
if err != nil {
return err
}
if err := migrations.CheckNotAhead(applied, ms); err != nil {
return err
}
n, err := migrations.Run(ctx, s.db, s.opts.Dialect, ms, s.log)
if err != nil {
return err
}
if n == 0 {
s.log.Debug("database schema is up to date", "migrations", len(ms))
}
return nil
}
// PendingMigrations implements storage.Database.
func (s *Store) PendingMigrations(ctx context.Context) ([]string, error) {
ms, err := s.migrations()
if err != nil {
return nil, err
}
pending, err := migrations.Pending(ctx, s.db, s.opts.Dialect, ms)
if err != nil {
return nil, err
}
out := make([]string, 0, len(pending))
for _, m := range pending {
out = append(out, fmt.Sprintf("%04d_%s", m.Version, m.Name))
}
return out, nil
}
func (s *Store) migrations() ([]migrations.Migration, error) {
return migrations.Load(s.opts.MigrationsFS, s.opts.MigrationsDir)
}